Skip to main content

subc_daemon/
supervise.rs

1use std::{
2    collections::{HashMap, HashSet, VecDeque},
3    error::Error,
4    fmt, io,
5    path::PathBuf,
6    process::{ExitStatus, Stdio},
7    sync::{Arc, Mutex, OnceLock},
8    time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19    manifest::{SelfSignalKind, SignalAnchor},
20    session::{
21        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22        MODULE_CONTROL_OP_HEALTH_CHECK,
23    },
24    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27    process::{Child, Command},
28    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29    task::JoinHandle,
30    time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35    child_roster::ChildRoster,
36    daemon_config::{
37        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38    },
39    forwarding::{
40        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41        ModuleDrainTarget, PendingModuleControlRpc,
42    },
43    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44    registry::{ConnectionId, RegistrationEndReason, RegistryError},
45    stderr_tail::{
46        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47        StderrTailSnapshot,
48    },
49    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50    Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62mod slow_retry;
63use slow_retry::{FailedRetry, FailureContext};
64
65#[cfg(all(test, unix))]
66mod slow_retry_tests;
67
68/// Command-line flag used by supervised modules to find subc.
69///
70/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
71/// The path points at the TCP+key connection file; it is not an ambient signal and
72/// is never inherited by standalone children.
73pub const SUBC_ARG: &str = "--subc";
74
75const DEFAULT_MAX_RESTARTS: u32 = 3;
76const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
77const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
78/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
79/// to contain a real crash loop (which respawns in seconds) and short enough
80/// that unrelated crashes hours apart never accumulate into a permanent stop.
81const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
82/// How long a drain waits for already-dispatched requests to finalize before
83/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
84/// not RPC-scale: the original 2s value silently cut nearly every real tool
85/// call at the fence, making the wait-for-finalize design decorative for the
86/// workloads it existed for. Quiescence short-circuits, so an idle module
87/// restarts immediately regardless of this value; the budget is spent only
88/// when a genuine in-flight request is worth finishing. Per-module override:
89/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
90/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
91/// where a stuck request will never settle).
92pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
93const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
94const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
95/// How long a drain waits after one busy-gauge probe completes before it asks
96/// again.
97///
98/// A drain re-probes on this short interval of its own rather than on the
99/// module's health cadence. The default cadence (30 s) equals the default
100/// drain ceiling, so a cadence-paced drain that found the module busy at its
101/// first probe never asked a second time: it waited out the whole ceiling and
102/// forced GOODBYE even after the gauges had read zero. The interval is counted
103/// from the moment the previous probe finished, and probes run one at a time,
104/// so a slow module is never asked more often than it can answer. Periodic
105/// health checks outside a drain keep using the configured cadence.
106const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
107/// How long a restart waits for the exited process's output readers.
108///
109/// The restart does not depend on either output reader finishing. A stderr
110/// reader still running at this bound is left running, and its later output goes
111/// into the exited process's own section of the stderr ring (see
112/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
113/// bound no longer decides whether a crash's last lines are kept: under load
114/// the reader may simply not have been scheduled yet, and cutting it there
115/// discarded exactly the lines that explained the crash.
116///
117/// What the bound still decides is when the tail starts reporting
118/// `Incomplete`: a pipe open past it usually means a descendant of the exited
119/// process still holds it, and the tail cannot claim to be whole until that
120/// pipe closes. That is also the only case in which the wait costs the restart
121/// anything, because a pipe with no other holder reaches EOF when the process
122/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
123/// `Captured` at EOF with nothing lost.
124///
125/// The stdout reader carries no ring, only the capture file. It also continues
126/// to EOF after this bound: child exit closes the writer but does not mean the
127/// reader has consumed all buffered bytes. Late output from either pipe can
128/// therefore reach the shared capture file after the successor starts.
129const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
130/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
131pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
132const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
133/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
134/// dropped because the subscriber stopped draining its frames. The detail's
135/// `first_undelivered_cursor` names the first event it did not receive; the
136/// client resubscribes from the last cursor it did receive.
137pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
138
139#[cfg(target_os = "macos")]
140struct PrivacyExec {
141    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
142    deadline: tokio::time::Instant,
143    expected: Option<subc_os::FileIdentity>,
144    trampoline: Option<subc_os::FileIdentity>,
145    script: bool,
146    module_id: String,
147    #[cfg(test)]
148    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
149}
150
151#[cfg(all(test, target_os = "macos"))]
152pub(crate) fn test_privacy_trampoline() -> PathBuf {
153    // Cargo's unit-test executable lives in <profile>/deps; its fixture bin
154    // lives beside that directory. This honors custom CARGO_TARGET_DIR too.
155    let path = std::env::current_exe()
156        .unwrap()
157        .parent()
158        .unwrap()
159        .parent()
160        .unwrap()
161        .join("privacy-trampoline-fixture");
162    // Without the fixture every macOS spawn is refused, and the tests that
163    // spawn fail later as a module in state Failed, which names the wrong
164    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
165    assert!(
166        path.exists(),
167        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
168         --bins --features test-support` or `cargo test -p subc-daemon` first",
169        path.display()
170    );
171    path
172}
173
174#[cfg(target_os = "macos")]
175fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
176    use std::io::Read;
177    let mut probe = std::process::Command::new(path)
178        .args(["__disclaim-exec", "--probe"])
179        .stdin(Stdio::null())
180        .stdout(Stdio::piped())
181        .stderr(Stdio::piped())
182        .spawn()
183        .map_err(|error| {
184            format!(
185                "privacy trampoline probe failed for {}: {error}",
186                path.display()
187            )
188        })?;
189    let deadline = std::time::Instant::now() + Duration::from_secs(5);
190    let status = loop {
191        match probe.try_wait() {
192            Ok(Some(status)) => break status,
193            Ok(None) if std::time::Instant::now() < deadline => {
194                std::thread::sleep(Duration::from_millis(5))
195            }
196            result => {
197                let _ = probe.kill();
198                let _ = probe.wait();
199                return Err(format!(
200                    "privacy trampoline probe failed or timed out for {}: {result:?}",
201                    path.display()
202                ));
203            }
204        }
205    };
206    let mut answer = String::new();
207    if let Some(stdout) = probe.stdout.take() {
208        let _ = stdout.take(256).read_to_string(&mut answer);
209    }
210    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
211        return Ok(());
212    }
213    let mut diagnostic = String::new();
214    if let Some(stderr) = probe.stderr.take() {
215        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
216    }
217    let cause = diagnostic
218        .trim()
219        .strip_prefix("ck-subc: own privacy identity refused: ")
220        .unwrap_or("binary does not implement the privacy trampoline protocol");
221    Err(format!(
222        "{cause}: probe of {} exited {status}",
223        path.display()
224    ))
225}
226
227#[cfg(target_os = "macos")]
228fn privacy_command(
229    spec: &ModuleSpec,
230    roster: &ChildRoster,
231) -> Result<
232    (
233        Command,
234        Option<PrivacyExec>,
235        subc_os::privacy_identity::ExecAcknowledgement,
236    ),
237    SuperviseError,
238> {
239    let failure = |cause: String| {
240        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
241        SuperviseError::Spawn {
242            program: spec.program.clone(),
243            source: io::Error::other(cause),
244            cgroup_path: None,
245        }
246    };
247    let trampoline = roster.privacy_trampoline().map_err(failure)?;
248    // Resolve PATH with the same environment the Command will receive. For
249    // scripts retain the existing orphan-identity rule: the kernel chooses
250    // the interpreter, and its observed image is the one recorded. Do not
251    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
252    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
253        let path = spec
254            .env
255            .iter()
256            .find(|(key, _)| key == "PATH")
257            .map(|(_, value)| std::ffi::OsString::from(value))
258            .or_else(|| std::env::var_os("PATH"))
259            .unwrap_or_else(|| "/usr/bin:/bin".into());
260        std::env::split_paths(&path)
261            .map(|dir| dir.join(&spec.program))
262            .find(|path| path.is_file())
263            .unwrap_or_else(|| spec.program.clone())
264    } else {
265        spec.program.clone()
266    };
267    let expected = subc_os::file_identity(&program);
268    let trampoline_image = subc_os::file_identity(&trampoline);
269    let script = {
270        use std::io::Read;
271        let mut prefix = [0u8; 2];
272        std::fs::File::open(&program)
273            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
274    };
275    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
276        return Err(failure(
277            "privacy identity module executable is missing or is the trampoline itself".to_string(),
278        ));
279    }
280    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
281        .map_err(|error| failure(error.to_string()))?;
282    let reader =
283        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
284    let mut command = Command::new(&trampoline);
285    command
286        .arg("__disclaim-exec")
287        .arg(ack.fd().to_string())
288        .arg(&program);
289    ack.install(command.as_std_mut());
290    Ok((
291        command,
292        Some(PrivacyExec {
293            reader,
294            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
295            expected,
296            trampoline: trampoline_image,
297            script,
298            module_id: spec.module_id.clone(),
299            #[cfg(test)]
300            confirmation_pause: spec
301                .env
302                .iter()
303                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
304                .map(|(_, address)| (address.clone(), None)),
305        }),
306        ack,
307    ))
308}
309
310struct SupervisedChild {
311    child: Child,
312    exec: crate::activity::ExecOwner,
313    #[cfg(windows)]
314    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
315    #[cfg(target_os = "macos")]
316    privacy_exec: Option<PrivacyExec>,
317    /// Set once this launch's exec acknowledgement confirms the module image.
318    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
319    /// `subc_os::privacy_identity`), which then replaces itself with the
320    /// module. The supervisor owns and can kill that pid from spawn, but
321    /// status readers report it only after this latch is set, so nothing
322    /// reports the trampoline's image as the module's.
323    #[cfg(target_os = "macos")]
324    report_ready: Arc<OnceLock<()>>,
325    /// Refusal before the module image was accepted, retained for terminal records.
326    spawn_failure: Option<String>,
327    /// The protocol this process was launched with. A reload can store a new
328    /// launch spec with a different protocol, but that takes effect only at the
329    /// next spawn, so this process keeps being handled by the protocol it
330    /// actually speaks.
331    protocol: ModuleProtocol,
332    /// This process's cgroup name: a bounded module/slot label followed by a
333    /// spawn suffix unique to this process (when cgroup placement is on). A
334    /// retired process in a slot may still be draining when a later one is
335    /// spawned into that slot, so the suffix keeps the later process out of
336    /// the retired one's cgroup, which is the domain a kill applies to.
337    #[cfg(target_os = "linux")]
338    module_id: String,
339    #[cfg(target_os = "linux")]
340    cgroup_placement: Option<subc_cgroup::Placement>,
341    /// The job that contains this child and every process it spawns (issue #109).
342    ///
343    /// Dropping this handle is what reaps a surviving tree when no supervisor
344    /// code runs — a daemon crash — because the job carries
345    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
346    ///
347    /// That limit is not crash-only, and the difference is worth knowing: a
348    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
349    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
350    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
351    /// module at once. Before this change they survived that, saw EOF on the
352    /// control socket, and ran their own teardown; Unix keeps that path
353    /// deliberately, so a module can seal a WAL or close a capture rather than
354    /// be killed mid-write. So this trades graceful teardown on every Windows
355    /// daemon stop for containment on a crash, which is the right way round
356    /// today: orphaned GPU workers are a reported, recurring problem, and the
357    /// modules that write most heavily do not run on Windows.
358    ///
359    /// The fix is a real Windows stop path — the daemon draining before it
360    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
361    /// reaches only what the drain left behind, which is what it should reach.
362    #[cfg(windows)]
363    job: Option<subc_jobobject::JobObject>,
364    stdout_pump: Option<JoinHandle<()>>,
365    stderr_pump: Option<StderrPump>,
366    stderr_ring: Arc<Mutex<StderrRing>>,
367    /// Conservative monotonic bound sampled before `command.spawn()`, not when
368    /// identity recording finishes or the spawn generation is published.
369    spawned_at: Instant,
370    spawned_at_ms: u64,
371    spawned_from: PathBuf,
372    launch_spec_digest: LaunchSpecDigest,
373    spawned_file_identity: Option<SpawnedFileIdentity>,
374    process_start_time: Option<u64>,
375    process_identity: Option<ProcessIdentity>,
376    pid: u32,
377    /// This process's entry in the daemon's child roster, released when the
378    /// process is reaped or this handle is dropped.
379    roster_guard: Option<crate::child_roster::RosterGuard>,
380}
381
382impl SupervisedChild {
383    fn id(&self) -> Option<u32> {
384        Some(self.pid)
385    }
386
387    fn process_identity(&self) -> Option<ProcessIdentity> {
388        self.process_identity
389    }
390
391    async fn wait(&mut self) -> io::Result<ExitStatus> {
392        #[cfg(target_os = "macos")]
393        self.confirm_privacy_exec().await;
394        // The roster entry is NOT released here. A daemon shutdown waits for the
395        // roster to empty and then exits the process, so releasing at the reap
396        // let it exit before the exit handler wrote this child's terminal record
397        // (the stderr drain and snapshot update sit in between), and the
398        // shutdown's own `daemon_shutdown` record was intermittently lost. The
399        // caller releases it after recording the exit (`release_roster`), and
400        // dropping the handle releases it too.
401        let result = self.child.wait().await;
402        #[cfg(target_os = "linux")]
403        if result.is_ok() {
404            if let Some(placement) = self.cgroup_placement.as_ref() {
405                cleanup_reaped_cgroup(placement, &self.module_id).await;
406                // Keep ownership while awaiting kernel population changes: a
407                // drain timeout may cancel this wait and then escalate/reap.
408                self.cgroup_placement = None;
409            }
410        }
411        result
412    }
413
414    #[cfg(target_os = "macos")]
415    async fn confirm_privacy_exec(&mut self) {
416        let Some(pending) = &mut self.privacy_exec else {
417            return;
418        };
419        let result = tokio::time::timeout_at(pending.deadline, async {
420            let mut record = Vec::new();
421            loop {
422                let mut ready = pending.reader.readable().await?;
423                let read = ready.try_io(|reader| {
424                    use std::io::Read;
425                    let mut reader = reader.get_ref();
426                    let mut buffer = [0u8; 256];
427                    reader.read(&mut buffer).map(|count| (count, buffer))
428                });
429                match read {
430                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
431                    Ok(Ok((count, buffer))) => {
432                        if record.len() + count > 1024 {
433                            return Err(io::Error::other(
434                                "privacy exec refusal record is too long",
435                            ));
436                        }
437                        record.extend_from_slice(&buffer[..count]);
438                    }
439                    Ok(Err(error)) => return Err(error),
440                    Err(_) => continue,
441                }
442            }
443        })
444        .await;
445        #[cfg(test)]
446        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
447            use tokio::io::{AsyncReadExt, AsyncWriteExt};
448            if stream.is_none() {
449                *stream = Some(
450                    tokio::net::TcpStream::connect(address.as_str())
451                        .await
452                        .unwrap(),
453                );
454                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
455                stream
456                    .as_mut()
457                    .unwrap()
458                    .write_all(&self.pid.to_ne_bytes())
459                    .await
460                    .unwrap();
461            }
462            let mut reply = [0];
463            stream
464                .as_mut()
465                .unwrap()
466                .read_exact(&mut reply)
467                .await
468                .unwrap();
469            pending.confirmation_pause = None;
470            if &reply == b"F" {
471                Ok(Err(io::Error::other("injected late exec refusal")))
472            } else {
473                assert_eq!(&reply, b"X");
474                result
475            }
476        } else {
477            result
478        };
479        // Keep the reader in self across await: select cancellation must not
480        // discard the handshake or reset its original five-second deadline.
481        let pending = self.privacy_exec.as_ref().expect("pending exec");
482        let cause = match result {
483            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
484            Ok(Err(error)) => Some(format!(
485                "privacy identity exec acknowledgement failed: {error}"
486            )),
487            Ok(Ok(record)) if !record.is_empty() => Some(
488                std::str::from_utf8(&record)
489                    .ok()
490                    .and_then(|record| {
491                        record
492                            .trim()
493                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
494                    })
495                    .filter(|cause| !cause.is_empty())
496                    .unwrap_or("invalid privacy exec refusal record")
497                    .to_string(),
498            ),
499            Ok(Ok(_)) => match self.child.try_wait() {
500                // Empty EOF is the exec acknowledgement. A real module may exit
501                // immediately, including with a reserved trampoline status; no
502                // image is admitted, and its ordinary exit contract stays intact.
503                Ok(Some(_status)) => None,
504                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
505                Ok(None) => {
506                    let image = observe_spawned_image(self.pid);
507                    if let Some(image) = image.filter(|image| {
508                        image.executable.is_some()
509                            && image.executable != pending.trampoline
510                            && (image.executable == pending.expected || pending.script)
511                    }) {
512                        if let Err(error) = self
513                            .roster_guard
514                            .as_ref()
515                            .map(|guard| guard.confirm_executable(image))
516                            .transpose()
517                        {
518                            Some(format!(
519                                "could not durably record confirmed module image: {error}"
520                            ))
521                        } else {
522                            let _ = self.report_ready.set(());
523                            info!(module_id = %pending.module_id, pid = self.pid,
524                                "module spawned with own privacy identity (responsibility disclaimed)");
525                            None
526                        }
527                    } else if image.is_none()
528                        || image.is_some_and(|image| image.executable.is_none())
529                    {
530                        // A process can exit between try_wait and the kernel
531                        // image read. Empty EOF already acknowledged exec, so
532                        // preserve that module's ordinary exit rather than
533                        // mislabel a disappearing image as trampoline refusal.
534                        // Keep pending in self across await so cancellation does
535                        // not discard validation or reset its original deadline.
536                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
537                            Ok(Ok(_status)) => None,
538                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
539                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
540                        }
541                    } else {
542                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
543                    }
544                }
545            },
546        };
547        let pending = self.privacy_exec.take().expect("pending exec");
548        // Exec confirmation describes launch success, not roster retirement.
549        // Keep even a reaped child recorded until its terminal history exists.
550        self.exec.0.finish(if cause.is_some() {
551            crate::activity::ExecState::Refused
552        } else {
553            crate::activity::ExecState::CommittedExec
554        });
555        if let Some(cause) = cause {
556            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
557            self.spawn_failure = Some(cause);
558            // No image is admitted on failure. Reach the entire fresh process
559            // group, including a module which spawned a helper before refusal.
560            if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
561                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
562            }
563            let _ = self.child.start_kill();
564        }
565    }
566
567    /// Releases this child's daemon-shutdown roster entry once its exit has
568    /// been recorded. The pid is already reaped and free for reuse, so the
569    /// entry must not outlive the record any longer than that.
570    fn release_roster(&mut self) {
571        self.roster_guard = None;
572    }
573
574    /// Kill the child and, where containment is available, its process tree.
575    ///
576    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
577    /// helper process leaked the helper — the Synapse embedding module's CUDA
578    /// worker holds the GPU allocation, so the leak cost VRAM until the next
579    /// restart of something else. Terminating the job reaches grandchildren that
580    /// a tree walk cannot, including one whose parent has already exited and
581    /// been reparented away.
582    ///
583    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
584    /// direct-child kill still decides the outcome, so containment can never
585    /// change whether a module is reported as stopped.
586    fn start_kill(&mut self) -> io::Result<()> {
587        #[cfg(windows)]
588        if let Some(job) = &self.job {
589            if let Err(error) = job.terminate() {
590                debug!(
591                    error = %error,
592                    "job termination failed; the direct-child kill still owns the outcome"
593                );
594            }
595        }
596        #[cfg(target_os = "linux")]
597        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
598        self.child.start_kill()
599    }
600
601    async fn drain_stderr(&mut self, module_id: &str) {
602        if let Some(pump) = self.stdout_pump.take() {
603            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
604        }
605
606        let Some(pump) = self.stderr_pump.take() else {
607            return;
608        };
609        settle_stderr_pump(
610            module_id,
611            &self.stderr_ring,
612            pump,
613            STDERR_PUMP_DRAIN_TIMEOUT,
614        )
615        .await;
616    }
617}
618
619/// Bound the restart's wait, not the lifetime of the exited child's stdout
620/// reader. Slow pipe reads or capture writes must not turn the bound into lost
621/// output; dropping the handle lets the reader finish at EOF, like stderr.
622async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
623    match timeout(bound, &mut pump).await {
624        Ok(Ok(())) => {}
625        Ok(Err(error)) => {
626            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
627        }
628        Err(_) => {
629            drop(pump);
630            warn!(
631                module_id,
632                waited = ?bound,
633                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
634            );
635        }
636    }
637}
638
639/// The reader task for one process's stderr, with the ring generation its
640/// lines are attributed to.
641struct StderrPump {
642    task: JoinHandle<()>,
643    generation: u64,
644}
645
646/// Retire an exited process's stderr reader and wait up to `bound` for it to
647/// reach EOF. A reader still running at the bound is detached, not stopped: it
648/// keeps filling the exited process's section of the ring until its pipe
649/// closes, and the tail reads `Incomplete` until then. See
650/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
651async fn settle_stderr_pump(
652    module_id: &str,
653    ring: &Arc<Mutex<StderrRing>>,
654    pump: StderrPump,
655    bound: Duration,
656) {
657    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
658    let StderrPump {
659        mut task,
660        generation,
661    } = pump;
662    lock().retire_pump(generation);
663    match timeout(bound, &mut task).await {
664        Ok(Ok(())) => {}
665        Ok(Err(err)) => {
666            let mut ring = lock();
667            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
668            ring.finish_pump(generation);
669            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
670        }
671        Err(_) => {
672            // Dropping the handle detaches the task; it ends at EOF on its pipe.
673            drop(task);
674            lock().mark_pump_late(
675                generation,
676                format!(
677                    "stderr of the exited process had not reached EOF {bound:?} after it was \
678                     retired (a descendant may still hold the pipe open); lines it still \
679                     writes are kept in that process's section"
680                ),
681            );
682            warn!(
683                module_id,
684                waited = ?bound,
685                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
686            );
687        }
688    }
689}
690
691fn registration_release_events() -> &'static watch::Sender<u64> {
692    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
693    EVENTS.get_or_init(|| {
694        let (sender, _receiver) = watch::channel(0);
695        sender
696    })
697}
698
699pub(crate) fn notify_registration_release() {
700    let events = registration_release_events();
701    let next_generation = (*events.borrow()).wrapping_add(1);
702    events.send_replace(next_generation);
703}
704
705/// How to launch one singleton module process.
706#[derive(Debug, Clone, PartialEq, Eq)]
707pub struct ModuleSpec {
708    pub module_id: String,
709    pub program: PathBuf,
710    pub args: Vec<String>,
711    pub env: Vec<(String, String)>,
712    /// When true this is a reserved module: each spawn gets a fresh one-time launch
713    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
714    /// process can register this module_id (a security-boundary module like the
715    /// credential vault must not be impersonable while it is down/restarting).
716    pub reserved: bool,
717    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
718    /// Prefixes come from daemon config and must end in `:` before they reach the
719    /// supervisor; the owner module's current spawn nonce authorizes claims under
720    /// each prefix.
721    pub reserved_prefixes: Vec<String>,
722    /// The wire protocol this module speaks, as DECLARED in daemon config.
723    ///
724    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
725    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
726    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
727    /// and NO launch nonce, and a clean exit the daemon did not request is
728    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
729    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
730    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
731    /// because a process ignores an environment variable it does not read.
732    ///
733    /// The argument is the part that cannot be "harmless to a process that
734    /// ignores it": a stock binary exits on an unknown flag before it listens
735    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
736    /// first conformance run against this mode found it. The nonce is withheld
737    /// because a process that will never present it gains nothing from holding
738    /// it, and a secret in the environment of a process that does not need it is
739    /// a leak surface for no benefit.
740    pub protocol: ModuleProtocol,
741    /// Whether two processes of this module may run at once, which is what a
742    /// blue/green swap does for the length of its overlap. Declared in daemon
743    /// config because the daemon must be able to answer it while the module is
744    /// down, and so a module cannot talk itself into it after registering.
745    pub overlap: ModuleOverlap,
746}
747
748/// Only the configured launch inputs enter this value; spawned connection paths,
749/// nonces and inherited environment never do. Its bytes remain daemon-private.
750#[derive(Clone, PartialEq, Eq)]
751pub(crate) struct LaunchSpecDigest([u8; 32]);
752
753impl fmt::Debug for LaunchSpecDigest {
754    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
755        f.write_str("LaunchSpecDigest([redacted])")
756    }
757}
758
759impl LaunchSpecDigest {
760    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
761        fn field(hash: &mut Sha256, bytes: &[u8]) {
762            hash.update((bytes.len() as u64).to_le_bytes());
763            hash.update(bytes);
764        }
765        let mut hash = Sha256::new();
766        if let Some(program) = spec.program.to_str() {
767            field(&mut hash, b"utf8-program");
768            field(&mut hash, program.as_bytes());
769        } else {
770            // The daemon config is UTF-8, but in-process callers may provide
771            // native paths: lossy conversion would collapse distinct programs.
772            #[cfg(unix)]
773            {
774                use std::os::unix::ffi::OsStrExt;
775                field(&mut hash, b"unix-program");
776                field(&mut hash, spec.program.as_os_str().as_bytes());
777            }
778            #[cfg(windows)]
779            {
780                use std::os::windows::ffi::OsStrExt;
781                field(&mut hash, b"windows-program");
782                let bytes = spec
783                    .program
784                    .as_os_str()
785                    .encode_wide()
786                    .flat_map(u16::to_le_bytes)
787                    .collect::<Vec<_>>();
788                field(&mut hash, &bytes);
789            }
790            #[cfg(not(any(unix, windows)))]
791            field(&mut hash, spec.program.to_string_lossy().as_bytes());
792        }
793        hash.update((spec.args.len() as u64).to_le_bytes());
794        for arg in &spec.args {
795            field(&mut hash, arg.as_bytes());
796        }
797        let mut env = spec.env.iter().collect::<Vec<_>>();
798        env.sort_unstable();
799        hash.update((env.len() as u64).to_le_bytes());
800        for (key, value) in env {
801            field(&mut hash, key.as_bytes());
802            field(&mut hash, value.as_bytes());
803        }
804        field(
805            &mut hash,
806            match spec.protocol {
807                ModuleProtocol::Subc => b"subc",
808                ModuleProtocol::None => b"none",
809            },
810        );
811        Self(hash.finalize().into())
812    }
813}
814
815/// Whether a module tolerates a second process of itself running alongside.
816///
817/// Most modules are single-writer on their store (a WAL, a capture log, a
818/// resident index behind a writer barrier), and two processes on one store
819/// corrupt it. So a swap, which overlaps the old and new process by design,
820/// is refused unless the module's config opts in with `overlap: "safe"`.
821#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
822pub enum ModuleOverlap {
823    /// Never run two processes of this module at once. The default.
824    #[default]
825    Exclusive,
826    /// The module has said a second process of itself is harmless for the
827    /// length of a swap.
828    ///
829    /// Declare it only if a second instance can run for a few seconds without
830    /// touching ANY single-writer store: every database, WAL, index, projector
831    /// and scheduled job the module owns. A lease on part of that state is not
832    /// enough. broca's session lease guards WAL appends while its run index, its
833    /// store projector and its archive fold timer (which unlinks live WAL files)
834    /// stay single-writer, so broca is exclusive despite holding a lease. The
835    /// refusal only fires after this has been decided, so the decision is the
836    /// check.
837    Safe,
838}
839
840impl ModuleOverlap {
841    pub fn as_str(self) -> &'static str {
842        match self {
843            Self::Exclusive => "exclusive",
844            Self::Safe => "safe",
845        }
846    }
847}
848
849/// Environment variable telling a spawned module which case it was started
850/// for, before it sends HELLO. Only a swap candidate carries it, as
851/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
852///
853/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
854/// longer because nobody waits on it, while a plain restart must flip ready
855/// quickly because callers see `module_warming` until it does. Absence means
856/// plain restart, the safe reading. The daemon trusts nothing about it; the
857/// candidate is proven by its launch nonce at HELLO.
858pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
859/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
860pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
861/// How long a swap waits for its candidate to register and declare itself
862/// ready when the operator does not say. A module warming as a swap candidate
863/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
864/// daemon allows that plus time to start the process and send HELLO.
865pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
866
867/// Bounded restart policy for crash exits.
868///
869/// `max_restarts` is the number of replacement processes allowed after the
870/// initial spawn WITHIN `window`. After that many crash restarts inside one
871/// window the module enters [`ModuleState::Failed`] and the supervisor stops
872/// the crash loop.
873///
874/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
875/// and that only survived because crashes were rare: a module that crashed
876/// three times across a week was disabled forever by crashes that had nothing
877/// to do with each other. That stopped being survivable once modules began
878/// exiting non-zero whenever the daemon's connection to them drops, because
879/// then every daemon-side connection drop spends a unit of the same budget and
880/// one flappy hour permanently stops a healthy module. Restarts older than
881/// `window` release their slot, so a module that crashed twice yesterday has a
882/// full budget today, while a genuine crash loop -- which is fast by
883/// definition -- still reaches the cap and stops.
884#[derive(Debug, Clone, Copy, PartialEq, Eq)]
885pub struct RestartPolicy {
886    pub max_restarts: u32,
887    /// Base delay before a crash replacement. The actual delay escalates with
888    /// the number of recent crash replacements and is capped by `max_backoff`.
889    pub backoff: Duration,
890    /// Maximum delay before a crash replacement.
891    pub max_backoff: Duration,
892    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
893    /// budget effectively infinite (nothing is ever in-window), which is why
894    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
895    pub window: Duration,
896}
897
898impl RestartPolicy {
899    /// A policy with the default crash window. Callers that care about the
900    /// window say so with [`Self::with_window`]; the ones that do not are
901    /// asking for the standard rate limit, not for no limit.
902    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
903        Self {
904            max_restarts,
905            backoff,
906            max_backoff: DEFAULT_MAX_BACKOFF,
907            window: DEFAULT_RESTART_WINDOW,
908        }
909    }
910
911    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
912        self.max_backoff = max_backoff;
913        self
914    }
915
916    pub fn with_window(mut self, window: Duration) -> Self {
917        self.window = window;
918        self
919    }
920
921    /// Calculate the capped exponential delay for the next crash replacement.
922    /// `restart_in_window` is zero for the first replacement after an operator
923    /// action (restart, reload, re-enable) cleared the crash ring, or after all
924    /// older crash replacements have aged out of the window.
925    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
926        if self.backoff.is_zero() || self.max_backoff.is_zero() {
927            return Duration::ZERO;
928        }
929
930        let mut delay = self.backoff;
931        for _ in 0..restart_in_window {
932            if delay >= self.max_backoff {
933                return self.max_backoff;
934            }
935            delay = delay
936                .checked_mul(10)
937                .unwrap_or(self.max_backoff)
938                .min(self.max_backoff);
939        }
940        delay.min(self.max_backoff)
941    }
942
943    /// The one sentence that explains a budget-exhausted stop, used for both the
944    /// log line and the terminal record so the two cannot drift. It names the
945    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
946    /// exactly what this budget is not.
947    fn budget_exhausted_detail(&self) -> String {
948        format!(
949            "crash budget exhausted: max_restarts={} within window_secs={}",
950            self.max_restarts,
951            self.window.as_secs()
952        )
953    }
954}
955
956impl Default for RestartPolicy {
957    fn default() -> Self {
958        Self {
959            max_restarts: DEFAULT_MAX_RESTARTS,
960            backoff: DEFAULT_BACKOFF,
961            max_backoff: DEFAULT_MAX_BACKOFF,
962            window: DEFAULT_RESTART_WINDOW,
963        }
964    }
965}
966
967#[derive(Debug, Clone, Copy, PartialEq, Eq)]
968struct CrashRestartSchedule {
969    restart_in_window: u32,
970    delay: Duration,
971}
972
973/// Whether the daemon itself will bring this module back after the exit being
974/// handled: it is enabled AND its in-window crash restarts are below the cap.
975///
976/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
977/// the window are dropped here rather than by a timer, so the count is right
978/// the moment somebody asks and no bookkeeping runs for idle modules.
979fn daemon_will_restart(
980    state: &mut SupervisorSnapshot,
981    policy: &RestartPolicy,
982    now: Instant,
983) -> bool {
984    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
985}
986
987const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
988const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
989const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
990const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
991
992#[derive(Debug, Clone, Copy, PartialEq, Eq)]
993pub enum HealthAction {
994    Report,
995    Restart,
996    Alert,
997}
998
999impl fmt::Display for HealthAction {
1000    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1001        f.write_str(match self {
1002            Self::Report => "report",
1003            Self::Restart => "restart",
1004            Self::Alert => "alert",
1005        })
1006    }
1007}
1008
1009#[derive(Debug, Clone, PartialEq, Eq)]
1010pub struct HealthConfig {
1011    /// Optional loopback HTTP endpoint for a managed non-wire process.
1012    /// Changing it applies live on rescan; the process protocol changes only
1013    /// at its next spawn.
1014    pub http: Option<String>,
1015    pub cadence: Duration,
1016    pub deadline: Duration,
1017    pub failure_threshold: u32,
1018    pub on_degraded: HealthAction,
1019    pub on_failing: HealthAction,
1020    pub critical: bool,
1021}
1022
1023impl Default for HealthConfig {
1024    fn default() -> Self {
1025        Self {
1026            http: None,
1027            cadence: DEFAULT_HEALTH_CADENCE,
1028            deadline: DEFAULT_HEALTH_DEADLINE,
1029            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1030            on_degraded: HealthAction::Report,
1031            on_failing: HealthAction::Report,
1032            critical: false,
1033        }
1034    }
1035}
1036
1037/// The supervisor's view of one module's health, relayed to clients over
1038/// channel-0 and rendered by `ck health`.
1039///
1040/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1041/// stated here rather than only at the wire type a consumer reads. A reader can
1042/// look up what `None` means; only a writer can silently change it, and the
1043/// writer has no reason to go looking at a downstream contract before editing.
1044///
1045/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1046/// back to `None` on re-registration precisely so a respawned module does not
1047/// carry its predecessor's timestamp — so an old value and an absent one call for
1048/// opposite readings, and anything that defaulted this to a number would make a
1049/// never-probed module indistinguishable from one probed at the epoch.
1050///
1051/// `detail` and `metrics` are `None` when the module published none on this
1052/// probe, which does not mean it reported nothing wrong — it is also the shape
1053/// when the probe never reached it. `last_probe_ms` is what separates those.
1054#[derive(Debug, Clone, PartialEq)]
1055pub struct ModuleHealthStatus {
1056    pub status: SupervisorHealthStatus,
1057    pub last_probe_ms: Option<u64>,
1058    pub detail: Option<String>,
1059    pub metrics: Option<Value>,
1060    pub consecutive_failures: u32,
1061    /// Number of replies received after a recurring health probe's deadline.
1062    /// Unlike a timeout, every increment proves the module was alive.
1063    pub late_answer_count: u64,
1064    /// End-to-end latency of the newest late reply, measured from probe start.
1065    pub last_late_answer_latency_ms: Option<u64>,
1066    pub last_action: Option<String>,
1067    /// Set together with `last_action`; the pair moves as one, and both being
1068    /// absent means no escalation has ever been taken rather than that the last
1069    /// one succeeded.
1070    pub last_action_ms: Option<u64>,
1071}
1072
1073impl Default for ModuleHealthStatus {
1074    fn default() -> Self {
1075        Self {
1076            status: SupervisorHealthStatus::Unknown,
1077            last_probe_ms: None,
1078            detail: None,
1079            metrics: None,
1080            consecutive_failures: 0,
1081            late_answer_count: 0,
1082            last_late_answer_latency_ms: None,
1083            last_action: None,
1084            last_action_ms: None,
1085        }
1086    }
1087}
1088
1089/// Typed lifecycle state for a supervised module.
1090#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1091pub enum ModuleState {
1092    Starting,
1093    Running,
1094    Unresponsive,
1095    Restarting,
1096    Draining,
1097    Stopped,
1098    Failed,
1099    Disabled,
1100}
1101
1102impl fmt::Display for ModuleState {
1103    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1104        f.write_str(match self {
1105            Self::Starting => "starting",
1106            Self::Running => "running",
1107            Self::Unresponsive => "unresponsive",
1108            Self::Restarting => "restarting",
1109            Self::Draining => "draining",
1110            Self::Stopped => "stopped",
1111            Self::Failed => "failed",
1112            Self::Disabled => "disabled",
1113        })
1114    }
1115}
1116
1117/// Supervisor classification of a child-process exit.
1118#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1119pub enum ExitKind {
1120    Clean,
1121    Crash,
1122    DeliberateSeverance,
1123}
1124
1125impl From<ExitKind> for TerminalExitKind {
1126    fn from(kind: ExitKind) -> Self {
1127        match kind {
1128            ExitKind::Clean => Self::Clean,
1129            ExitKind::Crash => Self::Crash,
1130            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1131        }
1132    }
1133}
1134
1135/// Exact process identity retained when a supervised module registers its
1136/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1137#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1138pub(crate) struct ProcessIdentity {
1139    pub(crate) pid: u32,
1140    pub(crate) start_time: u64,
1141}
1142
1143/// Last observed child exit, if any.
1144#[derive(Debug, Clone, PartialEq, Eq)]
1145pub struct ExitReport {
1146    pub kind: ExitKind,
1147    pub code: Option<i32>,
1148    pub signal: Option<i32>,
1149    pub at_ms: u64,
1150}
1151
1152/// Point-in-time module status answerable by subc without forwarding to the
1153/// module process.
1154#[derive(Debug, Clone, PartialEq)]
1155pub struct ModuleStatus {
1156    pub module_id: String,
1157    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1158    pub waiting_for_previous_pid: Option<u32>,
1159    /// Why the previous instance still prevents a new start.
1160    pub waiting_for_previous_detail: Option<String>,
1161    /// The daemon's Windows per-spawn record: `pipe` after the process it
1162    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1163    /// List/status reports use this record, not the source the module sends
1164    /// in its registration message's manifest.provenance.launch_nonce_source.
1165    pub launch_nonce_source: Option<String>,
1166    pub state: ModuleState,
1167    pub enabled: bool,
1168    pub process_alive: bool,
1169    pub registration_active: bool,
1170    /// The module's declared wire protocol, carried beside `live` because it is
1171    /// what makes `live` readable: the two fields answer one question together.
1172    /// While a process is alive this is its launch declaration, not a later
1173    /// pending-reload edit. When down it is the configured next launch protocol.
1174    pub protocol: ModuleProtocol,
1175    /// Whether the module is serving, under the strongest definition the daemon
1176    /// can assert for its protocol.
1177    ///
1178    /// A subc module must also be REGISTERED: its process being alive says
1179    /// nothing about whether it can take a request. A `protocol: "none"` module
1180    /// never registers, so that term is dropped and this falls back to "enabled,
1181    /// running, and the process the daemon launched is alive" -- which is all
1182    /// the daemon observes about a process that speaks no subc wire. It stays a
1183    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1184    /// rather than printing it bare.
1185    pub live: bool,
1186    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1187    /// restarts have already released their slot, so this count can go down
1188    /// without anybody touching the module.
1189    pub restart_count: u32,
1190    /// Replacement processes spawned over this module's entire supervisor lifetime;
1191    /// unlike `restart_count`, this value is never reset by an operator action
1192    /// and never falls out of a window.
1193    pub lifetime_restarts: u32,
1194    pub spawn_generation: u64,
1195    /// The budget `restart_count` is spent against. Carried alongside the count
1196    /// because the count alone does not say how close the module is to being
1197    /// disabled, and reporting one without the other is what makes an
1198    /// about-to-be-retired module look ordinary.
1199    pub max_restarts: u32,
1200    /// The span `restart_count` is counted over. Carried with the pair above for
1201    /// the same reason they are carried together: "2 of 3" means one thing for a
1202    /// ten-minute window and something else entirely for a lifetime.
1203    pub restart_window: Duration,
1204    /// Effective drain and restart timing policy used by this running module.
1205    /// These values are carried together with the restart budget so status
1206    /// readers can compare configured intent with what the supervisor applied.
1207    pub drain_timeout: Duration,
1208    pub restart_backoff: Duration,
1209    pub restart_max_backoff: Duration,
1210    /// The module's process. On macOS this stays absent while the `ck-subc`
1211    /// launch trampoline is still running in that pid, and appears once the
1212    /// exec acknowledgement confirms the module image has replaced it. Launch
1213    /// time and the supervisor's own hold on the process are unaffected.
1214    pub pid: Option<u32>,
1215    pub spawned_at_ms: Option<u64>,
1216    pub spawned_from: Option<PathBuf>,
1217    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1218    pub process_start_time: Option<u64>,
1219    pub last_exit: Option<ExitReport>,
1220    pub health: ModuleHealthStatus,
1221}
1222
1223#[cfg(test)]
1224#[derive(Debug, Clone, Copy, PartialEq)]
1225struct ActorSelectCheckpoint {
1226    generation: u64,
1227    turn: u64,
1228    registered_connection: Option<ConnectionId>,
1229    next_probe_at: Option<Instant>,
1230    wake_after: Option<Duration>,
1231}
1232
1233#[derive(Debug, Clone, PartialEq)]
1234struct SupervisorSnapshot {
1235    failed_retry: FailedRetry,
1236    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1237    #[cfg(windows)]
1238    pipe_consumed: Option<Arc<OnceLock<()>>>,
1239    /// Counts running-child loop turns, not executor polls of a parked wait.
1240    #[cfg(test)]
1241    actor_turns: u64,
1242    /// Running select after exec confirmation and with no unconsumed registry
1243    /// event. Recording a spawn or starting a loop turn is not this barrier.
1244    #[cfg(test)]
1245    actor_select: Option<ActorSelectCheckpoint>,
1246    state: ModuleState,
1247    enabled: bool,
1248    process_alive: bool,
1249    spawned_protocol: Option<ModuleProtocol>,
1250    spawn_failure: Option<String>,
1251    waiting_for_previous_pid: Option<u32>,
1252    waiting_for_previous_detail: Option<String>,
1253    /// When each crash restart was spent, oldest first. This IS the crash
1254    /// budget: its in-window length is the count an operator sees and the count
1255    /// the restart decision is made against, so there is no second counter that
1256    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1257    /// operator actions that used to zero the old lifetime counter.
1258    crash_restarts: VecDeque<Instant>,
1259    lifetime_restarts: u32,
1260    /// Successful child spawns in this daemon incarnation.
1261    ///
1262    /// `lifetime_restarts` was considered and rejected: it starts at zero
1263    /// (line 640), successful initial/operator spawns in `set_running` do not
1264    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1265    /// increments before a successful replacement exists (lines 604, 3846,
1266    /// and 3921), so a failed spawn can consume it. This counter moves only
1267    /// when a live PID is accepted below.
1268    spawn_generation: u64,
1269    pid: Option<u32>,
1270    #[cfg(target_os = "macos")]
1271    report_ready: Option<Arc<OnceLock<()>>>,
1272    /// Last reaped child, retained after current process facts are cleared.
1273    reaped_pid: Option<u32>,
1274    /// Whether a replacement has already been scheduled for the current process.
1275    respawn_pending: bool,
1276    /// A second restart is waiting for the replacement already scheduled.
1277    coalesced_restart_pending: bool,
1278    spawned_at_ms: Option<u64>,
1279    spawned_from: Option<PathBuf>,
1280    launch_spec_digest: Option<LaunchSpecDigest>,
1281    spawned_file_identity: Option<SpawnedFileIdentity>,
1282    process_start_time: Option<u64>,
1283    deliberate_severance: Option<ProcessIdentity>,
1284    last_exit: Option<ExitReport>,
1285    /// Diagnostic attached to the next drain's terminal record, if any.
1286    drain_disposition_detail: Option<String>,
1287    health: ModuleHealthStatus,
1288    /// Whether the current process was started as a swap candidate and so
1289    /// lives in the module's alternate cgroup. The next swap's candidate takes
1290    /// the other one, so the two processes of a swap never share a cgroup. A
1291    /// plain spawn always uses the primary cgroup.
1292    in_alternate_slot: bool,
1293    /// Whether the current `Draining` state ends in a replacement process
1294    /// (restart, reload, health restart) rather than a stop. Only meaningful
1295    /// while `state` is `Draining`; every entry into that state rewrites it.
1296    /// It is what lets route.open answer the retryable `module_reloading` to a
1297    /// consumer that reaches a still-registered process mid-restart, instead of
1298    /// the `supervisor_not_live` a stop or disable deserves.
1299    draining_to_replace: bool,
1300    /// Whether a configuration update has been applied since the current
1301    /// process was spawned, so that process runs an older spec than the one
1302    /// the supervisor now holds. A queued restart is only coalesced into a
1303    /// fresher process when this is false: a restart requested to pick up a
1304    /// new configuration must not be satisfied by a process that predates it.
1305    configuration_updated_since_spawn: bool,
1306}
1307
1308impl SupervisorSnapshot {
1309    /// The pid that status, provenance and resource readings may report. While
1310    /// the launch trampoline still runs in the pid, reading its executable or
1311    /// resource use would describe `ck-subc`, not the module, so none is
1312    /// reported until the exec acknowledgement confirms the module image.
1313    fn reported_pid(&self) -> Option<u32> {
1314        #[cfg(target_os = "macos")]
1315        if self
1316            .report_ready
1317            .as_ref()
1318            .is_some_and(|ready| ready.get().is_none())
1319        {
1320            return None;
1321        }
1322        self.pid
1323    }
1324
1325    fn starting() -> Self {
1326        Self::new(ModuleState::Starting, true)
1327    }
1328
1329    fn disabled() -> Self {
1330        Self::new(ModuleState::Disabled, false)
1331    }
1332
1333    /// Crash restarts still inside `window`, having dropped the ones that are
1334    /// not. Pruning on read is what makes the budget a rate: an instant older
1335    /// than the window stops holding a slot the moment anybody counts.
1336    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1337        while let Some(oldest) = self.crash_restarts.front() {
1338            if now.duration_since(*oldest) > window {
1339                self.crash_restarts.pop_front();
1340            } else {
1341                break;
1342            }
1343        }
1344        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1345    }
1346
1347    /// Spend one unit of the crash budget and record the restart in the ledger.
1348    ///
1349    /// The ring is bounded by the cap because more than `max_restarts` in-window
1350    /// instants can never be reached (the caller refuses the restart first), so
1351    /// anything beyond that is an unbounded queue waiting to happen.
1352    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1353        self.crash_restarts.push_back(now);
1354        while self.crash_restarts.len() > policy.max_restarts as usize {
1355            self.crash_restarts.pop_front();
1356        }
1357        self.lifetime_restarts += 1;
1358    }
1359
1360    /// Reserve one crash-restart slot and calculate the delay before respawning.
1361    /// The count is captured before recording this restart, so the first retry
1362    /// uses the base delay and each later in-window retry escalates once.
1363    fn next_crash_restart(
1364        &mut self,
1365        policy: &RestartPolicy,
1366        now: Instant,
1367    ) -> Option<CrashRestartSchedule> {
1368        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1369        if restart_in_window >= policy.max_restarts {
1370            return None;
1371        }
1372        self.record_crash_restart(policy, now);
1373        Some(CrashRestartSchedule {
1374            restart_in_window,
1375            delay: policy.delay_for_restart(restart_in_window),
1376        })
1377    }
1378
1379    /// Give the module its full budget back, as an operator restart, reload, or
1380    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1381    /// ledger of what actually happened, and an operator action does not unmake
1382    /// the crashes.
1383    fn clear_crash_restarts(&mut self) {
1384        self.crash_restarts.clear();
1385    }
1386
1387    fn new(state: ModuleState, enabled: bool) -> Self {
1388        Self {
1389            failed_retry: FailedRetry::default(),
1390            exec_outcome: None,
1391            #[cfg(test)]
1392            actor_turns: 0,
1393            #[cfg(test)]
1394            actor_select: None,
1395            state,
1396            enabled,
1397            process_alive: false,
1398            #[cfg(windows)]
1399            pipe_consumed: None,
1400            spawned_protocol: None,
1401            spawn_failure: None,
1402            waiting_for_previous_pid: None,
1403            waiting_for_previous_detail: None,
1404            crash_restarts: VecDeque::new(),
1405            lifetime_restarts: 0,
1406            spawn_generation: 0,
1407            pid: None,
1408            #[cfg(target_os = "macos")]
1409            report_ready: None,
1410            reaped_pid: None,
1411            respawn_pending: false,
1412            coalesced_restart_pending: false,
1413            spawned_at_ms: None,
1414            spawned_from: None,
1415            launch_spec_digest: None,
1416            spawned_file_identity: None,
1417            process_start_time: None,
1418            deliberate_severance: None,
1419            last_exit: None,
1420            drain_disposition_detail: None,
1421            health: ModuleHealthStatus::default(),
1422            in_alternate_slot: false,
1423            draining_to_replace: false,
1424            configuration_updated_since_spawn: false,
1425        }
1426    }
1427}
1428
1429type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1430
1431type SpawnSubscriberKey = (ConnectionId, u64);
1432
1433#[derive(Debug)]
1434struct SpawnSubscriber {
1435    version: u8,
1436    frames: mpsc::Sender<Frame>,
1437    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1438    /// from which event. The full frame channel cannot carry that news, so it
1439    /// travels beside it; see `SpawnEventFeed::subscribe`.
1440    lagged: Option<oneshot::Sender<SpawnCursor>>,
1441}
1442
1443#[derive(Debug)]
1444struct SpawnEventState {
1445    daemon_incarnation: String,
1446    seq: u64,
1447    capacity: usize,
1448    live: HashMap<String, LiveSpawn>,
1449    generations: HashMap<String, u64>,
1450    events: VecDeque<SpawnEvent>,
1451    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1452}
1453
1454impl Default for SpawnEventState {
1455    fn default() -> Self {
1456        Self {
1457            daemon_incarnation: "unconfigured".to_string(),
1458            seq: 0,
1459            capacity: SPAWN_EVENT_RING_CAPACITY,
1460            live: HashMap::new(),
1461            generations: HashMap::new(),
1462            events: VecDeque::new(),
1463            subscribers: HashMap::new(),
1464        }
1465    }
1466}
1467
1468#[derive(Debug, Clone, Default)]
1469struct SpawnEventFeed(
1470    Arc<Mutex<SpawnEventState>>,
1471    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1472);
1473
1474#[derive(Debug, Clone, PartialEq, Eq)]
1475pub(crate) enum SpawnSubscribeRefusal {
1476    ForeignIncarnation { current: String },
1477    TooOld { oldest: SpawnCursor },
1478    Frame(String),
1479}
1480
1481impl SpawnEventFeed {
1482    fn configure_incarnation(&self, daemon_incarnation: String) {
1483        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1484        state.daemon_incarnation = daemon_incarnation;
1485        state.seq = 0;
1486        state.live.clear();
1487        state.generations.clear();
1488        state.events.clear();
1489        state.subscribers.clear();
1490    }
1491
1492    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1493        SpawnCursor {
1494            daemon_incarnation: state.daemon_incarnation.clone(),
1495            seq: state.seq,
1496        }
1497    }
1498
1499    fn snapshot(&self) -> SpawnSnapshot {
1500        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1501        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1502        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1503        SpawnSnapshot {
1504            cursor: Self::cursor(&state),
1505            ring_bound: state.capacity as u64,
1506            live,
1507        }
1508    }
1509
1510    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1511        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1512        let generation = state
1513            .generations
1514            .get(module_id)
1515            .copied()
1516            .unwrap_or(0)
1517            .checked_add(1)
1518            .expect("spawn generation exhausted");
1519        state.generations.insert(module_id.to_string(), generation);
1520        let live = LiveSpawn {
1521            module_id: module_id.to_string(),
1522            spawn_generation: generation,
1523            pid,
1524            spawned_at_ms,
1525        };
1526        state.live.insert(module_id.to_string(), live);
1527        Self::emit_locked(
1528            &mut state,
1529            SpawnEventKind::Spawned,
1530            module_id.to_string(),
1531            generation,
1532            pid,
1533            None,
1534            None,
1535        );
1536        generation
1537    }
1538
1539    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1540        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1541        let Some(live) = state.live.remove(module_id) else {
1542            warn!(
1543                module_id,
1544                "terminal record had no live spawn event identity"
1545            );
1546            return;
1547        };
1548        Self::emit_locked(
1549            &mut state,
1550            SpawnEventKind::Exited,
1551            module_id.to_string(),
1552            live.spawn_generation,
1553            live.pid,
1554            exit_code,
1555            exit_signal,
1556        );
1557    }
1558
1559    /// Report the exit of a process that a swap has already replaced.
1560    ///
1561    /// `emit_exited` removes the module's live entry, which after a swap's
1562    /// cutover describes the promoted candidate, not the old process now
1563    /// exiting. This emits the old generation's exit and leaves the live entry
1564    /// alone unless it still names that generation.
1565    fn emit_superseded_exited(
1566        &self,
1567        module_id: &str,
1568        spawn_generation: u64,
1569        pid: u32,
1570        exit_code: Option<i32>,
1571        exit_signal: Option<i32>,
1572    ) {
1573        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1574        if state
1575            .live
1576            .get(module_id)
1577            .is_some_and(|live| live.spawn_generation == spawn_generation)
1578        {
1579            state.live.remove(module_id);
1580        }
1581        Self::emit_locked(
1582            &mut state,
1583            SpawnEventKind::Exited,
1584            module_id.to_string(),
1585            spawn_generation,
1586            pid,
1587            exit_code,
1588            exit_signal,
1589        );
1590    }
1591
1592    #[allow(clippy::too_many_arguments)]
1593    fn emit_locked(
1594        state: &mut SpawnEventState,
1595        kind: SpawnEventKind,
1596        module_id: String,
1597        spawn_generation: u64,
1598        pid: u32,
1599        exit_code: Option<i32>,
1600        exit_signal: Option<i32>,
1601    ) {
1602        state.seq = state
1603            .seq
1604            .checked_add(1)
1605            .expect("spawn event sequence exhausted");
1606        let event = SpawnEvent {
1607            cursor: Self::cursor(state),
1608            kind,
1609            module_id,
1610            spawn_generation,
1611            pid,
1612            exit_code,
1613            exit_signal,
1614        };
1615        state.events.push_back(event.clone());
1616        while state.events.len() > state.capacity {
1617            state.events.pop_front();
1618        }
1619        let body = match serde_json::to_vec(&event) {
1620            Ok(body) => body,
1621            Err(error) => {
1622                error!(%error, "failed to serialize supervisor spawn event");
1623                return;
1624            }
1625        };
1626        state.subscribers.retain(|(connection_id, corr), subscriber| {
1627            let frame = Frame::build_with_version(
1628                subscriber.version,
1629                FrameType::StreamData,
1630                control_flags(),
1631                0,
1632                0,
1633                *corr,
1634                body.clone(),
1635            );
1636            match frame {
1637                Ok(frame) => {
1638                    if subscriber.frames.try_send(frame).is_ok() {
1639                        true
1640                    } else {
1641                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1642                        if let Some(lagged) = subscriber.lagged.take() {
1643                            let _ = lagged.send(event.cursor.clone());
1644                        }
1645                        false
1646                    }
1647                }
1648                Err(error) => {
1649                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1650                    false
1651                }
1652            }
1653        });
1654    }
1655
1656    fn subscribe(
1657        &self,
1658        connection_id: ConnectionId,
1659        corr: u64,
1660        version: u8,
1661        since: Option<SpawnCursor>,
1662        sink: FrameSink,
1663    ) -> Result<(), SpawnSubscribeRefusal> {
1664        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1665        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1666        {
1667            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1668            let replay = if let Some(since) = since {
1669                if since.daemon_incarnation != state.daemon_incarnation {
1670                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1671                        current: state.daemon_incarnation.clone(),
1672                    });
1673                }
1674                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1675                    if since.seq < oldest.seq.saturating_sub(1) {
1676                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1677                    }
1678                }
1679                state
1680                    .events
1681                    .iter()
1682                    .filter(|event| event.cursor.seq > since.seq)
1683                    .cloned()
1684                    .collect::<Vec<_>>()
1685            } else {
1686                Vec::new()
1687            };
1688            for event in replay {
1689                let body = serde_json::to_vec(&event)
1690                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1691                let frame = Frame::build_with_version(
1692                    version,
1693                    FrameType::StreamData,
1694                    control_flags(),
1695                    0,
1696                    0,
1697                    corr,
1698                    body,
1699                )
1700                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1701                frames
1702                    .try_send(frame)
1703                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1704            }
1705            state.subscribers.insert(
1706                (connection_id, corr),
1707                SpawnSubscriber {
1708                    version,
1709                    frames,
1710                    lagged: Some(lagged),
1711                },
1712            );
1713        }
1714        // The lagged terminal is sent here, by the forwarder, rather than by
1715        // the emitter: at the moment of the drop the subscriber's own channel
1716        // is full, and writing to the connection sink directly from the emitter
1717        // would put the Error AHEAD of the events still queued in that channel
1718        // (and the emitter holds the feed lock, so it cannot await the sink).
1719        // Dropping the subscriber drops the only sender, so `recv` drains every
1720        // queued event and then returns `None`; only then is the Error sent, so
1721        // the client sees each event it can keep, then the reason it was cut.
1722        // Cancel and connection removal drop the oneshot unsent, so they end
1723        // the stream with no Error.
1724        tokio::spawn(async move {
1725            while let Some(frame) = receiver.recv().await {
1726                if sink.send(frame).await.is_err() {
1727                    return;
1728                }
1729            }
1730            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1731                return;
1732            };
1733            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1734                Ok(frame) => {
1735                    let _ = sink.send(frame).await;
1736                }
1737                Err(error) => {
1738                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1739                }
1740            }
1741        });
1742        Ok(())
1743    }
1744
1745    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1746        let Some(subscriber) = self
1747            .0
1748            .lock()
1749            .unwrap_or_else(|p| p.into_inner())
1750            .subscribers
1751            .remove(&(connection_id, corr))
1752        else {
1753            return false;
1754        };
1755        if let Ok(frame) = Frame::build_with_version(
1756            subscriber.version,
1757            FrameType::StreamEnd,
1758            control_flags(),
1759            0,
1760            0,
1761            corr,
1762            Vec::new(),
1763        ) {
1764            tokio::spawn(async move {
1765                let _ = subscriber.frames.send(frame).await;
1766            });
1767        }
1768        true
1769    }
1770
1771    fn remove_connection(&self, connection_id: ConnectionId) {
1772        self.0
1773            .lock()
1774            .unwrap_or_else(|p| p.into_inner())
1775            .subscribers
1776            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1777    }
1778
1779    #[cfg(any(test, feature = "test-support"))]
1780    fn set_capacity(&self, capacity: usize) {
1781        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1782    }
1783
1784    #[cfg(any(test, feature = "test-support"))]
1785    fn subscriber_count(&self) -> usize {
1786        self.0
1787            .lock()
1788            .unwrap_or_else(|p| p.into_inner())
1789            .subscribers
1790            .len()
1791    }
1792}
1793
1794/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1795/// The terminal Error a lagged spawn subscriber receives after its queued events.
1796fn spawn_subscriber_lagged_frame(
1797    version: u8,
1798    corr: u64,
1799    first_undelivered: SpawnCursor,
1800) -> Result<Frame, String> {
1801    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1802        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1803        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1804            .to_string(),
1805        detail: Some(serde_json::json!({
1806            "first_undelivered_cursor": first_undelivered
1807        })),
1808    })
1809    .map_err(|error| error.to_string())?;
1810    Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1811        .map_err(|error| error.to_string())
1812}
1813
1814pub trait ModuleProcessLiveness: Send + Sync {
1815    fn process_live(&self, module_id: &str) -> Option<bool>;
1816
1817    /// Whether the supervisor is replacing this module's process right now: an
1818    /// operator restart or reload, a health restart, or a crash respawn whose
1819    /// backoff is running. A module in that state is not live, but a consumer
1820    /// refused now should retry shortly rather than treat the target as gone.
1821    /// Stopped, failed, and disabled modules are not replacing.
1822    fn process_replacing(&self, _module_id: &str) -> bool {
1823        false
1824    }
1825}
1826
1827/// Shared process-liveness registry keyed by supervised `module_id`.
1828#[derive(Debug, Clone, Default)]
1829pub struct SupervisorProcessLiveness {
1830    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1831}
1832
1833impl SupervisorProcessLiveness {
1834    pub fn new() -> Self {
1835        Self::default()
1836    }
1837
1838    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1839        let mut snapshots = self
1840            .snapshots
1841            .lock()
1842            .unwrap_or_else(|poisoned| poisoned.into_inner());
1843        snapshots.insert(module_id, snapshot);
1844    }
1845
1846    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1847        let mut snapshots = self
1848            .snapshots
1849            .lock()
1850            .unwrap_or_else(|poisoned| poisoned.into_inner());
1851        let is_current = snapshots
1852            .get(module_id)
1853            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1854            .unwrap_or(false);
1855        if is_current {
1856            snapshots.remove(module_id);
1857        }
1858    }
1859}
1860
1861impl ModuleProcessLiveness for SupervisorProcessLiveness {
1862    fn process_live(&self, module_id: &str) -> Option<bool> {
1863        let snapshot = {
1864            let snapshots = self
1865                .snapshots
1866                .lock()
1867                .unwrap_or_else(|poisoned| poisoned.into_inner());
1868            snapshots.get(module_id).cloned()
1869        }?;
1870        let snapshot = snapshot
1871            .lock()
1872            .unwrap_or_else(|poisoned| poisoned.into_inner());
1873        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1874    }
1875
1876    fn process_replacing(&self, module_id: &str) -> bool {
1877        let Some(snapshot) = self
1878            .snapshots
1879            .lock()
1880            .unwrap_or_else(|poisoned| poisoned.into_inner())
1881            .get(module_id)
1882            .cloned()
1883        else {
1884            return false;
1885        };
1886        let snapshot = snapshot
1887            .lock()
1888            .unwrap_or_else(|poisoned| poisoned.into_inner());
1889        snapshot.enabled
1890            && match snapshot.state {
1891                ModuleState::Restarting => true,
1892                ModuleState::Draining => snapshot.draining_to_replace,
1893                ModuleState::Starting
1894                | ModuleState::Running
1895                | ModuleState::Unresponsive
1896                | ModuleState::Stopped
1897                | ModuleState::Failed
1898                | ModuleState::Disabled => false,
1899            }
1900    }
1901}
1902
1903#[cfg(test)]
1904#[derive(Debug, Default)]
1905struct ReloadExitRecordGate {
1906    reached: tokio::sync::Notify,
1907    resume: tokio::sync::Notify,
1908}
1909
1910#[derive(Debug, Clone, Copy)]
1911enum RespawnKind {
1912    Spawn,
1913    Reload,
1914}
1915
1916#[derive(Debug, Clone, Copy)]
1917struct PendingRespawn {
1918    deadline: Instant,
1919    kind: RespawnKind,
1920    daemon_initiated: bool,
1921}
1922
1923type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1924
1925#[derive(Debug, Clone)]
1926struct SupervisorRuntimeConfig {
1927    /// Retained across an operation's deferred backoff, until its child starts.
1928    recovery_is_daemon: bool,
1929    start_requester: Option<Principal>,
1930    pending_activity_restart: bool,
1931    provenance_probe: ExecutableIdentityProbe,
1932    /// Restart operations hand their backoff to the loop, which keeps serving commands.
1933    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1934    /// A reload acknowledges completion only after its replacement registers.
1935    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1936    restart_policy: RestartPolicy,
1937    /// This module's RESOLVED drain budget: per-module config when present,
1938    /// else `default_drain_timeout`.
1939    drain_timeout: Duration,
1940    /// Shared with the status handle so the attested value changes atomically
1941    /// when a rescan updates the running drain policy.
1942    effective_drain_timeout: Arc<Mutex<Duration>>,
1943    /// The supervisor-wide fallback, kept so a configuration update that
1944    /// REMOVES the per-module override can re-resolve to it.
1945    default_drain_timeout: Duration,
1946    health: HealthConfig,
1947    connection_file_path: Option<PathBuf>,
1948    capture_logs_dir: Option<PathBuf>,
1949    forwarding: Option<Arc<ForwardingTable>>,
1950    /// The shared handle, so every spawn path (initial, restart, reload) records the
1951    /// reserved-module launch nonce the HELLO verifier checks against.
1952    supervisor_handle: Option<SupervisorHandle>,
1953    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
1954    /// status queries.
1955    ///
1956    /// One ring per module, held across every respawn. The lines explaining an exit
1957    /// are written BEFORE that exit, so a ring recreated per process would be empty
1958    /// exactly when it is asked for.
1959    stderr_ring: Arc<Mutex<StderrRing>>,
1960    terminal_ring: Arc<Mutex<TerminalRing>>,
1961    spawn_events: SpawnEventFeed,
1962    child_roster: ChildRoster,
1963    #[cfg(target_os = "linux")]
1964    cgroup_placement: Option<subc_cgroup::Placement>,
1965    #[cfg(test)]
1966    test_seed_stale_facts_before_enable_spawn: bool,
1967    #[cfg(test)]
1968    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1969}
1970
1971impl SupervisorRuntimeConfig {
1972    fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
1973        FailureContext {
1974            daemon_initiated,
1975            shutdown: self.child_roster.shutdown_flag(),
1976        }
1977    }
1978}
1979
1980#[derive(Debug, Clone, PartialEq, Eq)]
1981struct SupervisedConfiguration {
1982    spec: ModuleSpec,
1983    health: HealthConfig,
1984}
1985
1986/// Shared daemon lookup table for supervised module handles.
1987///
1988/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
1989/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
1990/// launch nonces recorded at spawn are checked by the same daemon instance.
1991#[derive(Debug, Clone, Default)]
1992pub struct SupervisorHandle {
1993    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1994    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1995    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1996    /// Module ids the supervisor has taken on. An id is added BEFORE the
1997    /// module's first process is spawned and removed only when the module
1998    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
1999    /// the keys of `modules`.
2000    ///
2001    /// `modules` cannot answer "is this module configured?" on its own: a
2002    /// [`SupervisedModule`] only exists once its process has been spawned, and
2003    /// a fast child can connect, register, sync its scopes and ask about them
2004    /// before the supervisor has inserted it. Answering "not configured" in that
2005    /// gap makes scope admission refuse with the terminal "will never sync"
2006    /// instead of the retryable "has not synced yet".
2007    configured_ids: Arc<Mutex<HashSet<String>>>,
2008    spawn_events: SpawnEventFeed,
2009    /// The current expected launch nonce for each reserved module_id. Set when the
2010    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
2011    /// non-reserved module never has an entry here and is never nonce-checked.
2012    /// Reserved module ids and the nonce that authorizes their next HELLO.
2013    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
2014    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
2015    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
2016    /// had NO entry and admitted anyone: the reservation protected the nonce
2017    /// holder, not the NAME (found live by CKCRED's canary probe registering
2018    /// against a reserved scratch id).
2019    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2020    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2021    ///
2022    /// This is deliberately in-memory only: subc is state-free across daemon
2023    /// restarts, and the tombstone only explains the hours-after-removal window
2024    /// while this executing daemon is still alive. Do not persist it in a store.
2025    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2026    /// The current launch nonce for every supervised spawn. This is separate from
2027    /// reserved_nonces because consumer route.open attestation applies to all spawned
2028    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2029    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2030    /// Reserved namespace prefixes mapped to the supervised owner module whose
2031    /// current spawn nonce authorizes HELLO claims below the prefix.
2032    ///
2033    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2034    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2035    /// accidental collisions and lower-trust processes from squatting protected
2036    /// namespaces.
2037    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2038    /// Blue/green swaps in progress, by module id. An entry exists from just
2039    /// before the candidate process is spawned until the swap has failed, or
2040    /// has cut over and the old process is gone. While it exists, HELLO for the
2041    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2042    /// consumer attestation accepts both processes' nonces.
2043    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2044    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2045    promotion_observer: PromotionObserverSlot,
2046    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2047    /// this daemon-wide ordering, a rescan could retire or update a module while a
2048    /// concurrent reload still held its old handle and launch specification.
2049    operation_lock: Arc<AsyncMutex<()>>,
2050}
2051
2052/// Told when a swap has promoted its candidate to be the module's active
2053/// registration.
2054///
2055/// An ordinary HELLO runs the control plane's registration side effects (the
2056/// capability cache, the deny census, the requirement recompute) as it
2057/// registers. A swap candidate's HELLO does not, because it is not routable;
2058/// promotion is when those must run instead, and promotion happens in the
2059/// supervisor, which has no other way into the control handler.
2060pub(crate) trait SwapPromotionObserver: Send + Sync {
2061    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2062}
2063
2064/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2065/// control handler) owns this handle, so a strong reference back would be a
2066/// cycle that keeps both alive.
2067#[derive(Clone, Default)]
2068struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2069
2070impl fmt::Debug for PromotionObserverSlot {
2071    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2072        f.write_str("PromotionObserverSlot")
2073    }
2074}
2075
2076/// The nonces of one open swap.
2077#[derive(Debug, Clone)]
2078struct OpenSwap {
2079    /// The launch nonce minted for the candidate process. It is the swap
2080    /// token: the only thing that admits a HELLO into the candidate slot.
2081    candidate_nonce: String,
2082    /// The incumbent's launch nonce, captured when the swap opened. It is kept
2083    /// here because cutover moves the module's recorded spawn nonce to the
2084    /// candidate while the incumbent is still draining and its consumers are
2085    /// still attesting with this one.
2086    incumbent_nonce: Option<String>,
2087    /// Set once a HELLO has been admitted with the swap token, so the token
2088    /// admits one registration and cannot be replayed after cutover empties
2089    /// the candidate slot.
2090    candidate_admitted: bool,
2091}
2092
2093/// What the swap gate says about a HELLO. See
2094/// [`SupervisorHandle::swap_hello_admission`].
2095#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2096pub(crate) enum SwapHelloAdmission {
2097    /// No swap is open for the id (or the HELLO carries the incumbent's own
2098    /// nonce); the ordinary gates decide.
2099    NotSwapping,
2100    /// The HELLO carries the swap token: register it into the candidate slot.
2101    Candidate,
2102    /// A swap is open and the HELLO carries a nonce the supervisor did not
2103    /// mint for this id, no nonce, or a token already used.
2104    Refused,
2105}
2106
2107#[derive(Debug, Clone, PartialEq, Eq)]
2108pub(crate) enum ReservedHelloRejection {
2109    Exact {
2110        module_id: String,
2111    },
2112    Prefix {
2113        prefix: String,
2114        owner_module_id: String,
2115    },
2116}
2117
2118impl SupervisorHandle {
2119    pub fn new() -> Self {
2120        Self::default()
2121    }
2122
2123    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2124        self.spawn_events.snapshot()
2125    }
2126
2127    pub(crate) fn subscribe_spawns(
2128        &self,
2129        connection_id: ConnectionId,
2130        corr: u64,
2131        version: u8,
2132        since: Option<SpawnCursor>,
2133        sink: FrameSink,
2134    ) -> Result<(), SpawnSubscribeRefusal> {
2135        self.spawn_events
2136            .subscribe(connection_id, corr, version, since, sink)
2137    }
2138
2139    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2140        self.spawn_events.cancel(connection_id, corr)
2141    }
2142
2143    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2144        self.spawn_events.remove_connection(connection_id);
2145    }
2146
2147    #[cfg(any(test, feature = "test-support"))]
2148    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2149        assert!(capacity > 0, "spawn event capacity must be non-zero");
2150        self.spawn_events.set_capacity(capacity);
2151    }
2152
2153    #[cfg(any(test, feature = "test-support"))]
2154    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2155        self.spawn_events.subscriber_count()
2156    }
2157
2158    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2159    /// a respawn invalidates stale consumer identities.
2160    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2161        self.spawn_nonces
2162            .lock()
2163            .unwrap_or_else(|poisoned| poisoned.into_inner())
2164            .insert(module_id.to_string(), nonce);
2165    }
2166
2167    /// Record the launch nonce expected from the next HELLO for a reserved module,
2168    /// replacing any prior nonce (a respawn invalidates the previous one).
2169    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2170        self.reserved_nonces
2171            .lock()
2172            .unwrap_or_else(|poisoned| poisoned.into_inner())
2173            .insert(module_id.to_string(), Some(nonce));
2174    }
2175
2176    /// Record namespace prefixes owned by a supervised module.
2177    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2178        let mut owners = self
2179            .reserved_prefix_owners
2180            .lock()
2181            .unwrap_or_else(|poisoned| poisoned.into_inner());
2182        owners.retain(|_, owner| owner != owner_module_id);
2183        for prefix in prefixes {
2184            owners.insert(prefix.clone(), owner_module_id.to_string());
2185        }
2186    }
2187
2188    /// The launch nonce most recently minted for a module's spawn, if any.
2189    #[cfg(test)]
2190    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2191        self.spawn_nonces
2192            .lock()
2193            .unwrap_or_else(|poisoned| poisoned.into_inner())
2194            .get(module_id)
2195            .cloned()
2196    }
2197
2198    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2199        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2200        let spawn_nonce = self
2201            .spawn_nonces
2202            .lock()
2203            .unwrap_or_else(|poisoned| poisoned.into_inner())
2204            .get(&spec.module_id)
2205            .cloned();
2206        let mut reserved_nonces = self
2207            .reserved_nonces
2208            .lock()
2209            .unwrap_or_else(|poisoned| poisoned.into_inner());
2210        if spec.reserved {
2211            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2212            // reserved name whose module has never spawned has no legitimate
2213            // holder, and the entry's absence is what used to leave the name
2214            // open to the first claimant.
2215            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2216        }
2217        drop(reserved_nonces);
2218        // A later unreserved declaration must not silently unreserve an id that
2219        // was retained after its reserved configuration was removed. The explicit
2220        // release ceremony is the only operation that retires that gate.
2221        self.removal_tombstones
2222            .lock()
2223            .unwrap_or_else(|poisoned| poisoned.into_inner())
2224            .remove(&spec.module_id);
2225    }
2226
2227    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2228    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2229    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2230    /// with no matching prefix are always authorized.
2231    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2232        self.reserved_hello_verification(module_id, presented)
2233            .is_ok()
2234    }
2235
2236    /// Whether HELLO passed a nonce check that was actually required by reserved
2237    /// module policy, or the refusal explaining why a required check failed.
2238    pub(crate) fn reserved_hello_verification(
2239        &self,
2240        module_id: &str,
2241        presented: Option<&str>,
2242    ) -> Result<bool, ReservedHelloRejection> {
2243        let nonces = self
2244            .reserved_nonces
2245            .lock()
2246            .unwrap_or_else(|poisoned| poisoned.into_inner());
2247        if let Some(expected) = nonces.get(module_id) {
2248            // `None` = reserved with no legitimate holder: refuse every
2249            // presentation, because no process can hold a nonce that was never
2250            // minted. Only a real minted nonce admits, in constant time.
2251            let authorized = match expected {
2252                Some(expected) => {
2253                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2254                }
2255                None => false,
2256            };
2257            if authorized {
2258                return Ok(true);
2259            }
2260            return Err(ReservedHelloRejection::Exact {
2261                module_id: module_id.to_string(),
2262            });
2263        }
2264        drop(nonces);
2265
2266        let matched_prefix = self
2267            .reserved_prefix_owners
2268            .lock()
2269            .unwrap_or_else(|poisoned| poisoned.into_inner())
2270            .iter()
2271            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2272            .max_by_key(|(prefix, _)| prefix.len())
2273            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2274        let Some((prefix, owner_module_id)) = matched_prefix else {
2275            return Ok(false);
2276        };
2277
2278        let authorized = presented.is_some_and(|presented| {
2279            self.spawn_nonces
2280                .lock()
2281                .unwrap_or_else(|poisoned| poisoned.into_inner())
2282                .get(&owner_module_id)
2283                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2284                // While the owner is being swapped, children started by
2285                // either of its two processes hold that process's nonce.
2286                || self.swap_nonce_matches(&owner_module_id, presented)
2287        });
2288        if authorized {
2289            Ok(true)
2290        } else {
2291            Err(ReservedHelloRejection::Prefix {
2292                prefix,
2293                owner_module_id,
2294            })
2295        }
2296    }
2297
2298    /// Whether a consumer connection proved it came from a daemon-spawned module.
2299    ///
2300    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2301    /// accepted only for module ids the supervisor has spawned.
2302    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2303        if presented.is_empty() {
2304            return false;
2305        }
2306        let nonces = self
2307            .spawn_nonces
2308            .lock()
2309            .unwrap_or_else(|poisoned| poisoned.into_inner());
2310        let current = nonces
2311            .get(module_id)
2312            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2313        drop(nonces);
2314        // During a swap two processes of the module are alive, and a consumer
2315        // started by either one presents that process's nonce. Accepting only
2316        // the recorded one would fail the incumbent's consumers for the whole
2317        // overlap once cutover moves the record to the candidate.
2318        current || self.swap_nonce_matches(module_id, presented)
2319    }
2320
2321    /// Whether `presented` is either nonce of an open swap for `module_id`.
2322    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2323        let swaps = self
2324            .swaps
2325            .lock()
2326            .unwrap_or_else(|poisoned| poisoned.into_inner());
2327        swaps.get(module_id).is_some_and(|swap| {
2328            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2329                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2330                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2331                })
2332        })
2333    }
2334
2335    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2336    /// Called before the candidate process exists.
2337    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2338        let incumbent_nonce = self
2339            .spawn_nonces
2340            .lock()
2341            .unwrap_or_else(|poisoned| poisoned.into_inner())
2342            .get(module_id)
2343            .cloned();
2344        self.swaps
2345            .lock()
2346            .unwrap_or_else(|poisoned| poisoned.into_inner())
2347            .insert(
2348                module_id.to_string(),
2349                OpenSwap {
2350                    candidate_nonce,
2351                    incumbent_nonce,
2352                    candidate_admitted: false,
2353                },
2354            );
2355    }
2356
2357    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2358    /// the module's recorded one.
2359    pub(crate) fn close_swap(&self, module_id: &str) {
2360        self.swaps
2361            .lock()
2362            .unwrap_or_else(|poisoned| poisoned.into_inner())
2363            .remove(module_id);
2364    }
2365
2366    /// Install the observer told about swap promotions, replacing any earlier
2367    /// one.
2368    pub(crate) fn set_swap_promotion_observer(
2369        &self,
2370        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2371    ) {
2372        *self
2373            .promotion_observer
2374            .0
2375            .lock()
2376            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2377    }
2378
2379    /// Tell the installed observer, if it is still alive, that a swap promoted
2380    /// `registration`.
2381    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2382        let observer = self
2383            .promotion_observer
2384            .0
2385            .lock()
2386            .unwrap_or_else(|poisoned| poisoned.into_inner())
2387            .as_ref()
2388            .and_then(std::sync::Weak::upgrade);
2389        if let Some(observer) = observer {
2390            observer.swap_promoted(registration);
2391        }
2392    }
2393
2394    /// Whether a swap is open for `module_id`.
2395    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2396        self.swaps
2397            .lock()
2398            .unwrap_or_else(|poisoned| poisoned.into_inner())
2399            .contains_key(module_id)
2400    }
2401
2402    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2403    /// respawn would, once cutover has made the candidate the module's process.
2404    /// The swap stays open so the incumbent's nonce keeps attesting until the
2405    /// incumbent has drained and exited.
2406    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2407        let candidate_nonce = self
2408            .swaps
2409            .lock()
2410            .unwrap_or_else(|poisoned| poisoned.into_inner())
2411            .get(module_id)
2412            .map(|swap| swap.candidate_nonce.clone());
2413        let Some(nonce) = candidate_nonce else {
2414            return;
2415        };
2416        self.set_spawn_nonce(module_id, nonce.clone());
2417        if reserved {
2418            self.set_reserved_nonce(module_id, nonce);
2419        }
2420    }
2421
2422    /// The swap gate for a HELLO claiming `module_id`.
2423    ///
2424    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2425    /// presents the candidate nonce, which the reserved gate (holding the
2426    /// incumbent's nonce) would refuse as `reserved_module` before swap
2427    /// admission was ever reached. And it applies to unreserved ids too: for an
2428    /// unreserved id the only thing that ever stopped a second process claiming
2429    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2430    /// refusal a swap lifts for its candidate.
2431    ///
2432    /// The incumbent's own nonce falls through to the ordinary gates, which
2433    /// treat it as they always have (a live incumbent is refused as a
2434    /// duplicate). Anything else while a swap is open is refused, including an
2435    /// absent nonce.
2436    pub(crate) fn swap_hello_admission(
2437        &self,
2438        module_id: &str,
2439        presented: Option<&str>,
2440    ) -> SwapHelloAdmission {
2441        let swaps = self
2442            .swaps
2443            .lock()
2444            .unwrap_or_else(|poisoned| poisoned.into_inner());
2445        let Some(swap) = swaps.get(module_id) else {
2446            return SwapHelloAdmission::NotSwapping;
2447        };
2448        let Some(presented) = presented else {
2449            return SwapHelloAdmission::Refused;
2450        };
2451        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2452            return if swap.candidate_admitted {
2453                SwapHelloAdmission::Refused
2454            } else {
2455                SwapHelloAdmission::Candidate
2456            };
2457        }
2458        if swap
2459            .incumbent_nonce
2460            .as_deref()
2461            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2462        {
2463            return SwapHelloAdmission::NotSwapping;
2464        }
2465        SwapHelloAdmission::Refused
2466    }
2467
2468    /// Record that the swap token has registered a candidate, so it admits no
2469    /// second HELLO.
2470    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2471        if let Some(swap) = self
2472            .swaps
2473            .lock()
2474            .unwrap_or_else(|poisoned| poisoned.into_inner())
2475            .get_mut(module_id)
2476        {
2477            swap.candidate_admitted = true;
2478        }
2479    }
2480
2481    /// Test/support lookup for the current launch nonce of a supervised spawn.
2482    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2483        self.spawn_nonces
2484            .lock()
2485            .unwrap_or_else(|poisoned| poisoned.into_inner())
2486            .get(module_id)
2487            .cloned()
2488    }
2489
2490    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2491    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2492        self.reserved_nonces
2493            .lock()
2494            .unwrap_or_else(|poisoned| poisoned.into_inner())
2495            .get(module_id)
2496            .cloned()
2497            .flatten()
2498    }
2499
2500    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2501        // Normally already marked before the process was spawned; marking here
2502        // too keeps `configured_ids` a superset of the roster for any caller
2503        // that inserts a module directly.
2504        self.mark_configured(module.module_id());
2505        let mut modules = self
2506            .modules
2507            .lock()
2508            .unwrap_or_else(|poisoned| poisoned.into_inner());
2509        modules.insert(module.module_id().to_string(), module)
2510    }
2511
2512    /// Record that the supervisor has taken on `module_id`. Called before the
2513    /// module's first process is spawned, so that by the time that process can
2514    /// register, [`Self::is_configured`] already answers true.
2515    fn mark_configured(&self, module_id: &str) {
2516        self.configured_ids
2517            .lock()
2518            .unwrap_or_else(|poisoned| poisoned.into_inner())
2519            .insert(module_id.to_string());
2520    }
2521
2522    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2523    /// before it was ever put on the roster. A module already on the roster
2524    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2525    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2526        let modules = self
2527            .modules
2528            .lock()
2529            .unwrap_or_else(|poisoned| poisoned.into_inner());
2530        if !modules.contains_key(module_id) {
2531            self.configured_ids
2532                .lock()
2533                .unwrap_or_else(|poisoned| poisoned.into_inner())
2534                .remove(module_id);
2535        }
2536    }
2537
2538    /// Whether `module_id` is a module this daemon supervises: on the roster,
2539    /// or about to be (its process is being spawned right now).
2540    ///
2541    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2542    /// for scopes: a supervised module's process can register and sync before
2543    /// [`Self::get`] can return it, and in that window it is still a module
2544    /// that will sync, not one that never will.
2545    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2546        self.configured_ids
2547            .lock()
2548            .unwrap_or_else(|poisoned| poisoned.into_inner())
2549            .contains(module_id)
2550    }
2551
2552    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2553        let modules = self
2554            .modules
2555            .lock()
2556            .unwrap_or_else(|poisoned| poisoned.into_inner());
2557        modules.get(module_id).cloned()
2558    }
2559
2560    pub(crate) fn record_late_health_answer(
2561        &self,
2562        module_id: &str,
2563        latency_ms: u64,
2564    ) -> Result<bool, SuperviseError> {
2565        let Some(module) = self.get(module_id) else {
2566            return Ok(false);
2567        };
2568        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2569            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2570            state.health.last_late_answer_latency_ms = Some(latency_ms);
2571            // A late answer is an answer: the module served the probe, just past
2572            // the deadline. Leaving the miss streak in place while logging
2573            // "proves the module is alive" is how a CPU-starved module that
2574            // answers every probe a few seconds late still marches to the
2575            // threshold and gets killed — the exact kill class `NoAnswer` is
2576            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2577            // is degradation, and degradation reports; it does not restart.
2578            state.health.consecutive_failures = 0;
2579        })?;
2580        Ok(true)
2581    }
2582
2583    /// Arm the one-shot marker for the module process that this caller
2584    /// deliberately initiated severance against. Generic connection teardown
2585    /// must not call this:
2586    /// a surviving process would otherwise retain an exemption for a later
2587    /// genuine crash.
2588    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2589        let Some(module) = self.get(module_id) else {
2590            return Ok(false);
2591        };
2592        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2593        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2594            return Ok(false);
2595        };
2596        drop(snapshot);
2597        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2598    }
2599
2600    pub fn list(&self) -> Vec<SupervisedModule> {
2601        let modules = self
2602            .modules
2603            .lock()
2604            .unwrap_or_else(|poisoned| poisoned.into_inner());
2605        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2606        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2607        modules
2608    }
2609
2610    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2611        self.activity_images
2612            .lock()
2613            .unwrap_or_else(|p| p.into_inner())
2614            .remove(module_id);
2615        self.spawn_nonces
2616            .lock()
2617            .unwrap_or_else(|poisoned| poisoned.into_inner())
2618            .remove(module_id);
2619        self.close_swap(module_id);
2620        let mut reserved_nonces = self
2621            .reserved_nonces
2622            .lock()
2623            .unwrap_or_else(|poisoned| poisoned.into_inner());
2624        if reserved_nonces.contains_key(module_id) {
2625            // The old nonce must die with the removed process, but the exact-id
2626            // gate remains until an operator explicitly releases it.
2627            reserved_nonces.insert(module_id.to_string(), None);
2628        }
2629        drop(reserved_nonces);
2630        self.reserved_prefix_owners
2631            .lock()
2632            .unwrap_or_else(|poisoned| poisoned.into_inner())
2633            .retain(|_, owner| owner != module_id);
2634        let removed = self
2635            .modules
2636            .lock()
2637            .unwrap_or_else(|poisoned| poisoned.into_inner())
2638            .remove(module_id);
2639        self.configured_ids
2640            .lock()
2641            .unwrap_or_else(|poisoned| poisoned.into_inner())
2642            .remove(module_id);
2643        removed
2644    }
2645
2646    /// Remember a module removed by a non-preview rescan so route.open can
2647    /// distinguish that intentional removal from an unknown id.
2648    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2649        self.removal_tombstones
2650            .lock()
2651            .unwrap_or_else(|poisoned| poisoned.into_inner())
2652            .insert(module_id.to_string(), unix_ms_now());
2653    }
2654
2655    /// Return how long ago a rescan removed this module in milliseconds.
2656    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2657        self.removal_tombstones
2658            .lock()
2659            .unwrap_or_else(|poisoned| poisoned.into_inner())
2660            .get(module_id)
2661            .copied()
2662            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2663    }
2664
2665    /// Retire a reserved-id gate only after its module has left supervision.
2666    ///
2667    /// A retained gate has no live nonce (`None`), so releasing any other entry
2668    /// would weaken a currently configured or otherwise active reservation.
2669    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2670        if self.get(module_id).is_some() {
2671            return false;
2672        }
2673        let mut reserved_nonces = self
2674            .reserved_nonces
2675            .lock()
2676            .unwrap_or_else(|poisoned| poisoned.into_inner());
2677        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2678            return false;
2679        }
2680        reserved_nonces.remove(module_id);
2681        true
2682    }
2683
2684    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2685        Arc::clone(&self.operation_lock)
2686    }
2687}
2688
2689/// Process supervisor for subc-owned singleton modules.
2690#[derive(Debug, Clone)]
2691pub struct Supervisor {
2692    registry: Arc<Registry>,
2693    restart_policy: RestartPolicy,
2694    drain_timeout: Duration,
2695    connection_file_path: Option<PathBuf>,
2696    capture_logs_dir: Option<PathBuf>,
2697    forwarding: Option<Arc<ForwardingTable>>,
2698    process_liveness: Arc<SupervisorProcessLiveness>,
2699    supervisor_handle: Option<SupervisorHandle>,
2700    health: HealthConfig,
2701    daemon_start_clock: crate::clock::StartClock,
2702    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2703    spawn_events: SpawnEventFeed,
2704    provenance_probe: ExecutableIdentityProbe,
2705    /// Every process spawned through this supervisor (and its clones) and not
2706    /// yet reaped, so daemon shutdown can end them.
2707    child_roster: ChildRoster,
2708    #[cfg(target_os = "linux")]
2709    cgroup_placement: Option<subc_cgroup::Placement>,
2710    #[cfg(test)]
2711    test_after_first_spawn: AfterFirstSpawnHook,
2712}
2713
2714/// A roster entry whose monitor and first process have not started yet.
2715/// Keeping the command receiver here lets discovery serve status immediately
2716/// while start/stop requests wait until bootstrap has published the connection file.
2717pub(crate) struct RegisteredConfiguredModule {
2718    spec: ModuleSpec,
2719    runtime: SupervisorRuntimeConfig,
2720    module: SupervisedModule,
2721    commands: mpsc::Receiver<SupervisorCommand>,
2722}
2723
2724/// Test-only hook run on the path that takes on a new module, right after its
2725/// first `spawn_child` returns (the process exists and could already be
2726/// registering) and before that process is handed to the module's supervise
2727/// loop and put on the roster. Lets a test observe what a fast child would see
2728/// in that window without racing a real one.
2729#[cfg(test)]
2730#[derive(Clone, Default)]
2731struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2732
2733#[cfg(test)]
2734type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2735
2736#[cfg(test)]
2737impl fmt::Debug for AfterFirstSpawnHook {
2738    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2739        f.write_str("AfterFirstSpawnHook")
2740    }
2741}
2742
2743#[cfg(test)]
2744impl AfterFirstSpawnHook {
2745    fn run(&self, module_id: &str) {
2746        if let Some(hook) = &self.0 {
2747            hook(module_id);
2748        }
2749    }
2750}
2751
2752impl Supervisor {
2753    #[cfg(test)]
2754    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2755        let supervisor = Self::new(registry, policy);
2756        #[cfg(target_os = "macos")]
2757        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2758        supervisor
2759    }
2760    /// Verify the trampoline once when configured. Missing private OS support
2761    /// refuses every macOS launch by name but does not stop the daemon's control
2762    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2763    /// entry point; the library must not exec an arbitrary hosting program.
2764    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2765        let path = path.into();
2766        #[cfg(target_os = "macos")]
2767        {
2768            let result = probe_privacy_trampoline(&path).map(|()| path);
2769            if let Err(cause) = &result {
2770                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2771            }
2772            self.child_roster.set_privacy_trampoline(result);
2773        }
2774        #[cfg(not(target_os = "macos"))]
2775        let _ = path;
2776        self
2777    }
2778    /// The flag [`Self::begin_daemon_shutdown`] sets, for the control handler's
2779    /// capability-requirement log to read.
2780    pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2781        self.child_roster.shutdown_flag()
2782    }
2783
2784    /// The first step of an announced daemon shutdown, before the notice and
2785    /// before any connection is closed.
2786    ///
2787    /// Sets the daemon-shutdown flag first: from here on no module is
2788    /// respawned (crash restart, operator restart, or swap), and every child
2789    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2790    /// the module exits on the EOF this shutdown gives it or is signalled by a
2791    /// service manager that kills the whole cgroup. Then writes the journal's
2792    /// shutdown marker, which records the instant and closes this daemon
2793    /// incarnation's stretch of the journal.
2794    #[cfg(unix)]
2795    pub(crate) fn begin_daemon_shutdown(&self) {
2796        self.child_roster.close();
2797        if let Some(journal) = &self.terminal_journal {
2798            journal.stamp_shutdown();
2799        }
2800    }
2801
2802    /// Announce a cut while established connections can still carry replies.
2803    /// These budgets promise notice and a bounded wait, not child completion;
2804    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2805    #[cfg(unix)]
2806    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2807        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2808        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2809        let Some(forwarding) = &self.forwarding else {
2810            return Ok(());
2811        };
2812        let notice_started = Instant::now();
2813        let modules = forwarding
2814            .begin_daemon_drain()
2815            .map_err(SuperviseError::Forwarding)?;
2816        let deadline_ms =
2817            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2818        let mut notices = tokio::task::JoinSet::new();
2819        let mut drains = Vec::new();
2820        for (module_id, endpoint) in modules {
2821            let Some(target) = forwarding
2822                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2823                .map_err(SuperviseError::Forwarding)?
2824            else {
2825                continue;
2826            };
2827            let routes = forwarding
2828                .endpoint_routes(target.endpoint)
2829                .map_err(SuperviseError::Forwarding)?;
2830            // Restart allows deployed consumers to reopen after the new daemon
2831            // appears. The wire reason stays `restart`; what tells a daemon cut
2832            // apart from a module restart afterwards is the terminal record
2833            // itself, whose disposition is `daemon_shutdown` for every exit
2834            // observed once `begin_daemon_shutdown` has run.
2835            let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2836                reason: RouteCloseReason::Restart,
2837                deadline_ms,
2838            })
2839            .expect("module draining serializes");
2840            let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2841            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2842            for route in routes {
2843                let client = route.goodbye_target;
2844                if let Some((_, channels)) = clients
2845                    .iter_mut()
2846                    .find(|(existing, _)| existing.connection_id == client.connection_id)
2847                {
2848                    channels.push(client.channel);
2849                } else {
2850                    let channel = client.channel;
2851                    clients.push((client, vec![channel]));
2852                }
2853            }
2854            for (client, mut channels) in clients {
2855                channels.sort_unstable();
2856                channels.dedup();
2857                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2858                    module_id: module_id.clone(),
2859                    channels,
2860                    reason: RouteCloseReason::Restart,
2861                })
2862                .expect("route closing serializes");
2863                recipients.push((client.sink, client.negotiated_ver, closing));
2864            }
2865            for (sink, version, body) in recipients {
2866                notices.spawn(async move {
2867                    let frame = Frame::build_with_version(
2868                        version,
2869                        FrameType::Push,
2870                        control_flags(),
2871                        0,
2872                        0,
2873                        0,
2874                        body,
2875                    )
2876                    .expect("bounded lifecycle notice frame builds");
2877                    sink.send_flushed(frame).await
2878                });
2879            }
2880            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2881            drains.push((module_id, target.endpoint, gauges));
2882        }
2883        // A quiet forwarding table is not proof that queued notices reached the
2884        // socket. Wait for writer flush acknowledgements before testing quiescence.
2885        let notice_deadline = Instant::now() + NOTICE_BUDGET;
2886        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2887            if !matches!(result, Ok(Ok(()))) {
2888                warn!(?result, "daemon shutdown notice delivery failed");
2889            }
2890        }
2891        notices.abort_all();
2892        let notice_elapsed = notice_started.elapsed();
2893        let drain_started = Instant::now();
2894        let deadline = drain_started + DRAIN_BUDGET;
2895        let mut waits = tokio::task::JoinSet::new();
2896        for (module_id, endpoint, gauges) in drains {
2897            let forwarding = Arc::clone(forwarding);
2898            let mut runtime = self.runtime_config();
2899            runtime.health.cadence = Duration::from_millis(100);
2900            waits.spawn(async move {
2901                let wait_result = wait_for_forwarding_quiescence(
2902                    &forwarding,
2903                    &module_id,
2904                    &runtime,
2905                    endpoint,
2906                    deadline,
2907                    &gauges,
2908                    DrainScope::Endpoint(endpoint),
2909                    &mut DrainGaugeTrace::new(&gauges),
2910                )
2911                .await;
2912                // Store the measurement before the task returns: quiescence
2913                // may be reached just as the deadline stops the join loop from
2914                // collecting results. Unfinished or failed waits report false.
2915                forwarding
2916                    .record_daemon_drain_result(
2917                        endpoint,
2918                        drained_after_quiescence_wait(&wait_result),
2919                    )
2920                    .map_err(SuperviseError::Forwarding)?;
2921                wait_result
2922            });
2923        }
2924        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2925            if !matches!(result, Ok(Ok(true))) {
2926                warn!(?result, "daemon shutdown drain did not reach quiescence");
2927            }
2928        }
2929        // Measure only the forwarding drain, not child teardown or the caller's
2930        // signal delivery and process-reaping delays.
2931        info!(
2932            notice_elapsed_ms = notice_elapsed.as_millis() as u64,
2933            drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
2934            "daemon shutdown forwarding drain complete"
2935        );
2936        Ok(())
2937    }
2938
2939    /// The last step of an announced daemon shutdown, after the notice and the
2940    /// drain: send every registered module a module GOODBYE, the same planned
2941    /// stop signal `ck module stop` gives, then close every connection so each
2942    /// subc module sees EOF and starts its own teardown, then end every
2943    /// supervised child that has not exited
2944    /// by its own deadline (its drain budget, capped). Modules lead their own
2945    /// process groups, so a
2946    /// service manager's group kill no longer reaches them; without this a
2947    /// child that does not stop on EOF (every `protocol: "none"` child, which
2948    /// has no connection) would outlive the daemon. Every wait is bounded (see
2949    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
2950    #[cfg(unix)]
2951    pub(crate) async fn end_children_for_daemon_shutdown(
2952        &self,
2953        already_escalated: bool,
2954        escalate: impl std::future::Future<Output = ()>,
2955    ) {
2956        tokio::pin!(escalate);
2957        let mut escalated = already_escalated;
2958        if let Some(forwarding) = &self.forwarding {
2959            let reason = CloseReason::new(
2960                "daemon_shutdown",
2961                "the daemon is exiting after its shutdown notice and drain",
2962            );
2963            if escalated {
2964                // The operator asked to stop waiting: queue the GOODBYEs but
2965                // do not wait for them to be written.
2966                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2967            } else {
2968                tokio::select! {
2969                    biased;
2970                    _ = escalate.as_mut() => {
2971                        info!("second SIGTERM: abandoning module GOODBYE delivery");
2972                        escalated = true;
2973                    }
2974                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2975                }
2976            }
2977            let closed = forwarding.close_all_connections(&reason);
2978            debug!(closed, "closed established connections for daemon shutdown");
2979        }
2980        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
2981        // already completed and must not be polled again; the child shutdown
2982        // wait is told it is escalated and gets a future that never fires.
2983        let escalated_here = escalated && !already_escalated;
2984        let remaining_escalate = async move {
2985            if escalated_here {
2986                std::future::pending::<()>().await;
2987            } else {
2988                escalate.await;
2989            }
2990        };
2991        crate::child_roster::end_children_for_daemon_shutdown(
2992            &self.child_roster,
2993            escalated,
2994            remaining_escalate,
2995        )
2996        .await;
2997    }
2998
2999    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3000        Self {
3001            registry,
3002            restart_policy,
3003            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3004            connection_file_path: None,
3005            capture_logs_dir: None,
3006            forwarding: None,
3007            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3008            supervisor_handle: None,
3009            health: HealthConfig::default(),
3010            daemon_start_clock: crate::clock::StartClock::capture(),
3011            terminal_journal: None,
3012            spawn_events: SpawnEventFeed::default(),
3013            provenance_probe: ExecutableIdentityProbe::default(),
3014            child_roster: ChildRoster::default(),
3015            #[cfg(target_os = "linux")]
3016            cgroup_placement: None,
3017            #[cfg(test)]
3018            test_after_first_spawn: AfterFirstSpawnHook::default(),
3019        }
3020    }
3021
3022    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3023        self.drain_timeout = drain_timeout;
3024        self
3025    }
3026
3027    /// The drain budget a module gets when its config entry sets none.
3028    pub(crate) fn default_drain_timeout(&self) -> Duration {
3029        self.drain_timeout
3030    }
3031
3032    pub fn with_process_liveness(
3033        mut self,
3034        process_liveness: Arc<SupervisorProcessLiveness>,
3035    ) -> Self {
3036        self.process_liveness = process_liveness;
3037        self
3038    }
3039
3040    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3041        self.connection_file_path = Some(connection_file_path.into());
3042        self
3043    }
3044
3045    /// Enables daemon-owned capture files for supervised stdout and stderr.
3046    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3047        self.capture_logs_dir = Some(logs_dir.into());
3048        self
3049    }
3050
3051    /// Names this daemon lifetime in spawn events, independently of whether a
3052    /// terminal journal is configured.
3053    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3054        // A millisecond start stamp can repeat after clock rollback or a rapid
3055        // restart. Use the connection file's random daemon_id instead: it already
3056        // identifies this daemon lifetime independently of the wall clock.
3057        self.spawn_events.configure_incarnation(daemon_incarnation);
3058        self
3059    }
3060
3061    /// Enables best-effort history shared by every supervised module. Without
3062    /// it, terminal history is kept only in each module's in-memory ring.
3063    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3064        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3065        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3066            path,
3067            daemon_incarnation,
3068        )));
3069        this
3070    }
3071
3072    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3073        self.forwarding = Some(forwarding);
3074        self
3075    }
3076
3077    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3078        self.spawn_events = supervisor_handle.spawn_events.clone();
3079        self.supervisor_handle = Some(supervisor_handle);
3080        self
3081    }
3082
3083    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3084        self.health = health;
3085        self
3086    }
3087
3088    /// Keeps a record of every live child at `path`, rewritten on each spawn and
3089    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
3090    /// record is kept.
3091    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3092        self.child_roster.record_to(path.into());
3093        self
3094    }
3095
3096    #[cfg(target_os = "linux")]
3097    pub fn with_cgroup_placement(
3098        mut self,
3099        cgroup_placement: Option<subc_cgroup::Placement>,
3100    ) -> Self {
3101        self.cgroup_placement = cgroup_placement;
3102        self
3103    }
3104
3105    /// Spawn `spec.program` and start monitoring it.
3106    ///
3107    /// The child is expected to parse `--subc <connection-file-path>`, read the
3108    /// TCP+key connection file, authenticate to the already-running listener, and
3109    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
3110    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3111        validate_spec(&spec)?;
3112        self.establish_identity(&spec);
3113
3114        let runtime = self.runtime_config();
3115        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3116        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3117        let spawned = spawn_child(
3118            &spec,
3119            runtime.connection_file_path.as_deref(),
3120            self.supervisor_handle.as_ref(),
3121            &runtime.stderr_ring,
3122            runtime.capture_logs_dir.as_deref(),
3123            &runtime.child_roster,
3124            #[cfg(target_os = "linux")]
3125            runtime.cgroup_placement.as_ref(),
3126        );
3127        #[cfg(test)]
3128        self.test_after_first_spawn.run(&spec.module_id);
3129        let child = match spawned {
3130            Ok(child) => child,
3131            Err(err) => {
3132                // Unlike the configured paths, a failed `spawn` leaves nothing
3133                // on the roster, so the module must not stay marked configured.
3134                self.abandon_unrostered(&spec.module_id);
3135                return Err(err);
3136            }
3137        };
3138        self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3139        if let Some(spawn) = activity_spawn {
3140            spawn.complete(&snapshot);
3141        }
3142
3143        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3144    }
3145
3146    /// Make `spec`'s module count as configured, with its identity gates
3147    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3148    /// exists.
3149    ///
3150    /// Every path that takes on a new module calls this before `spawn_child`.
3151    /// The order is the point: the child can connect, register, sync its
3152    /// scopes and ask about them as soon as it is spawned, and the module is
3153    /// only put on the roster after `spawn_child` returns. Were the mark set
3154    /// with the roster entry, a fast child would see its own owner reported
3155    /// as not configured, and a scoped `route.open` in that window would be
3156    /// refused as terminal `scope_not_live` ("will never sync") instead of
3157    /// retryable `scope_not_synced`.
3158    fn establish_identity(&self, spec: &ModuleSpec) {
3159        if let Some(supervisor_handle) = &self.supervisor_handle {
3160            supervisor_handle.apply_identity_configuration(spec);
3161            supervisor_handle.mark_configured(&spec.module_id);
3162        }
3163    }
3164
3165    /// Take back [`Self::establish_identity`]'s configured mark when the
3166    /// module will not be put on the roster after all.
3167    fn abandon_unrostered(&self, module_id: &str) {
3168        if let Some(supervisor_handle) = &self.supervisor_handle {
3169            supervisor_handle.unmark_configured_unless_rostered(module_id);
3170        }
3171    }
3172
3173    /// Record a freshly spawned first process as running. On failure the
3174    /// module never reaches the roster, so its configured mark is taken back.
3175    fn mark_first_process_running(
3176        &self,
3177        spec: &ModuleSpec,
3178        runtime: &SupervisorRuntimeConfig,
3179        snapshot: &SharedSnapshot,
3180        child: &SupervisedChild,
3181    ) -> Result<(), SuperviseError> {
3182        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3183            self.abandon_unrostered(&spec.module_id);
3184            return Err(err);
3185        }
3186        self.process_liveness
3187            .track(spec.module_id.clone(), Arc::clone(snapshot));
3188        Ok(())
3189    }
3190
3191    /// Start supervising a module declared in daemon configuration.
3192    ///
3193    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3194    /// failures in the supervisor handle so operator-facing `supervisor.list`
3195    /// reflects every configured module while daemon startup continues.
3196    pub fn supervise_configured(
3197        &self,
3198        spec: ModuleSpec,
3199        enabled: bool,
3200    ) -> Result<SupervisedModule, SuperviseError> {
3201        self.supervise_configured_requested(
3202            spec,
3203            enabled,
3204            self.health.clone(),
3205            None,
3206            self.restart_policy,
3207            None,
3208        )
3209    }
3210
3211    /// Supervise a configured module with its own health, drain, and crash
3212    /// budget. The restart policy is per-module because the config file is:
3213    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3214    /// module that is expensive to restart should not be forced onto the same
3215    /// budget as one that is cheap.
3216    pub fn supervise_configured_with_health(
3217        &self,
3218        spec: ModuleSpec,
3219        enabled: bool,
3220        health: HealthConfig,
3221        drain_timeout_ms: Option<u64>,
3222        restart_policy: RestartPolicy,
3223    ) -> Result<SupervisedModule, SuperviseError> {
3224        self.supervise_configured_requested(
3225            spec,
3226            enabled,
3227            health,
3228            drain_timeout_ms,
3229            restart_policy,
3230            None,
3231        )
3232    }
3233
3234    pub(crate) fn supervise_configured_requested(
3235        &self,
3236        spec: ModuleSpec,
3237        enabled: bool,
3238        health: HealthConfig,
3239        drain_timeout_ms: Option<u64>,
3240        restart_policy: RestartPolicy,
3241        requester: Option<Principal>,
3242    ) -> Result<SupervisedModule, SuperviseError> {
3243        let registered = self.register_configured_with_health(
3244            spec,
3245            enabled,
3246            health,
3247            drain_timeout_ms,
3248            restart_policy,
3249        )?;
3250        let daemon_initiated = requester.is_none();
3251        self.start_registered_configured(registered, requester, daemon_initiated)
3252    }
3253
3254    /// Record a configured module as starting or disabled, without launching a
3255    /// process or running its monitor. Bootstrap registers the entire roster
3256    /// before publishing discovery, then activates it once children can read
3257    /// the connection file passed in their `--subc` argument.
3258    pub(crate) fn register_configured_with_health(
3259        &self,
3260        spec: ModuleSpec,
3261        enabled: bool,
3262        health: HealthConfig,
3263        drain_timeout_ms: Option<u64>,
3264        restart_policy: RestartPolicy,
3265    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3266        validate_spec(&spec)?;
3267        self.establish_identity(&spec);
3268
3269        let mut runtime = self.runtime_config();
3270        runtime.health = health;
3271        runtime.restart_policy = restart_policy;
3272        if let Some(ms) = drain_timeout_ms {
3273            runtime.drain_timeout = Duration::from_millis(ms);
3274            *runtime
3275                .effective_drain_timeout
3276                .lock()
3277                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3278        }
3279        let snapshot = if enabled {
3280            SupervisorSnapshot::starting()
3281        } else {
3282            SupervisorSnapshot::disabled()
3283        };
3284        let snapshot = {
3285            let mut snapshot = snapshot;
3286            if enabled {
3287                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3288                {
3289                    snapshot.waiting_for_previous_pid = Some(pid);
3290                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3291                }
3292            }
3293            snapshot
3294        };
3295        let (module, commands) =
3296            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3297        Ok(RegisteredConfiguredModule {
3298            spec,
3299            runtime,
3300            module,
3301            commands,
3302        })
3303    }
3304
3305    /// Activate a previously registered module. Queued control requests are
3306    /// served by the monitor after the initial configured spawn has settled.
3307    pub(crate) fn start_registered_configured(
3308        &self,
3309        registered: RegisteredConfiguredModule,
3310        requester: Option<Principal>,
3311        daemon_initiated: bool,
3312    ) -> Result<SupervisedModule, SuperviseError> {
3313        let RegisteredConfiguredModule {
3314            spec,
3315            mut runtime,
3316            module,
3317            commands,
3318        } = registered;
3319        runtime.start_requester = requester;
3320        runtime.recovery_is_daemon = daemon_initiated;
3321        let snapshot = &module.inner.snapshot;
3322        let should_spawn = {
3323            let state = lock_snapshot(snapshot)?;
3324            state.enabled && state.waiting_for_previous_pid.is_none()
3325        };
3326        let child = if should_spawn {
3327            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3328            let spawned = spawn_child(
3329                &spec,
3330                runtime.connection_file_path.as_deref(),
3331                self.supervisor_handle.as_ref(),
3332                &runtime.stderr_ring,
3333                runtime.capture_logs_dir.as_deref(),
3334                &runtime.child_roster,
3335                #[cfg(target_os = "linux")]
3336                runtime.cgroup_placement.as_ref(),
3337            );
3338            #[cfg(test)]
3339            self.test_after_first_spawn.run(&spec.module_id);
3340            match spawned {
3341                Ok(child) => {
3342                    self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3343                    if let Some(spawn) = activity_spawn {
3344                        spawn.complete(snapshot);
3345                    }
3346                    Some(child)
3347                }
3348                Err(err) => {
3349                    if runtime.health.critical {
3350                        error!(
3351                            module_id = %spec.module_id,
3352                            program = %spec.program.display(),
3353                            error = %err,
3354                            "critical configured module failed to spawn; marking failed and alerting"
3355                        );
3356                    } else {
3357                        error!(
3358                            module_id = %spec.module_id,
3359                            program = %spec.program.display(),
3360                            error = %err,
3361                            "configured module failed to spawn; marking failed and continuing"
3362                        );
3363                    }
3364                    fail_snapshot(
3365                        snapshot,
3366                        Some(&spec.module_id),
3367                        None,
3368                        &runtime.failure_context(daemon_initiated),
3369                    );
3370                    None
3371                }
3372            }
3373        } else {
3374            None
3375        };
3376        self.start_monitor(spec, runtime, &module, child, commands);
3377        Ok(module)
3378    }
3379
3380    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3381        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3382        SupervisorRuntimeConfig {
3383            recovery_is_daemon: true,
3384            start_requester: None,
3385            pending_activity_restart: false,
3386            provenance_probe: self.provenance_probe.clone(),
3387            scheduled_respawn: Arc::default(),
3388            deferred_reload_reply: Arc::default(),
3389            restart_policy: self.restart_policy,
3390            drain_timeout: self.drain_timeout,
3391            // Shared with this module's roster copy: daemon shutdown waits on
3392            // each child for the module's own drain budget, as resolved now.
3393            child_roster: self
3394                .child_roster
3395                .for_module(Arc::clone(&effective_drain_timeout)),
3396            effective_drain_timeout,
3397            default_drain_timeout: self.drain_timeout,
3398            health: self.health.clone(),
3399            connection_file_path: self.connection_file_path.clone(),
3400            capture_logs_dir: self.capture_logs_dir.clone(),
3401            forwarding: self.forwarding.clone(),
3402            supervisor_handle: self.supervisor_handle.clone(),
3403            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3404            terminal_ring: Arc::new(Mutex::new(
3405                TerminalRing::new(
3406                    TerminalRingConfig::default(),
3407                    self.daemon_start_clock.started_at_ms(),
3408                )
3409                .with_start_clock(self.daemon_start_clock)
3410                .with_journal(self.terminal_journal.clone())
3411                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3412            )),
3413            spawn_events: self.spawn_events.clone(),
3414            #[cfg(target_os = "linux")]
3415            cgroup_placement: self.cgroup_placement.clone(),
3416            #[cfg(test)]
3417            test_seed_stale_facts_before_enable_spawn: false,
3418            #[cfg(test)]
3419            test_reload_exit_record_gate: None,
3420        }
3421    }
3422
3423    fn supervised_module(
3424        &self,
3425        spec: ModuleSpec,
3426        runtime: SupervisorRuntimeConfig,
3427        snapshot: SharedSnapshot,
3428        child: Option<SupervisedChild>,
3429    ) -> SupervisedModule {
3430        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3431        self.start_monitor(spec, runtime, &module, child, commands);
3432        module
3433    }
3434
3435    fn register_module(
3436        &self,
3437        spec: &ModuleSpec,
3438        runtime: &SupervisorRuntimeConfig,
3439        snapshot: SharedSnapshot,
3440    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3441        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3442            spec: spec.clone(),
3443            health: runtime.health.clone(),
3444        }));
3445        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3446        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3447        // The module's OWN policy, which may be its per-module config rather than
3448        // the supervisor-wide one; status must report the budget the supervise
3449        // loop actually enforces.
3450        let restart_policy = runtime.restart_policy;
3451        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3452        let (tx, rx) = mpsc::channel(4);
3453        let module_id = spec.module_id.clone();
3454        let module = SupervisedModule {
3455            inner: Arc::new(SupervisedModuleInner {
3456                module_id: module_id.clone(),
3457                registry: Arc::clone(&self.registry),
3458                snapshot,
3459                configuration,
3460                stderr_ring,
3461                terminal_ring,
3462                commands: tx,
3463                monitor: Mutex::new(None),
3464                restart_policy,
3465                effective_drain_timeout,
3466                provenance_probe: self.provenance_probe.clone(),
3467            }),
3468        };
3469        // Identity gates and the configured mark must precede the roster entry,
3470        // so a client never sees a module without its admission policy.
3471        if let Some(supervisor_handle) = &self.supervisor_handle {
3472            supervisor_handle.insert(module.clone());
3473        }
3474        (module, rx)
3475    }
3476
3477    fn start_monitor(
3478        &self,
3479        spec: ModuleSpec,
3480        runtime: SupervisorRuntimeConfig,
3481        module: &SupervisedModule,
3482        child: Option<SupervisedChild>,
3483        commands: mpsc::Receiver<SupervisorCommand>,
3484    ) {
3485        let monitor = tokio::spawn(supervise_loop(
3486            spec,
3487            runtime,
3488            Arc::clone(&self.registry),
3489            Arc::clone(&self.process_liveness),
3490            Arc::clone(&module.inner.snapshot),
3491            child,
3492            commands,
3493        ));
3494        *module
3495            .inner
3496            .monitor
3497            .lock()
3498            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3499    }
3500}
3501
3502impl Default for Supervisor {
3503    fn default() -> Self {
3504        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3505    }
3506}
3507
3508/// Handle to one supervised child process.
3509#[derive(Clone)]
3510pub struct SupervisedModule {
3511    inner: Arc<SupervisedModuleInner>,
3512}
3513
3514struct SupervisedModuleInner {
3515    module_id: String,
3516    registry: Arc<Registry>,
3517    snapshot: SharedSnapshot,
3518    configuration: Arc<Mutex<SupervisedConfiguration>>,
3519    stderr_ring: Arc<Mutex<StderrRing>>,
3520    terminal_ring: Arc<Mutex<TerminalRing>>,
3521    commands: mpsc::Sender<SupervisorCommand>,
3522    monitor: Mutex<Option<JoinHandle<()>>>,
3523    /// Copied from the supervisor's runtime config at spawn so `status()` can
3524    /// report the restart budget without reaching back into the supervisor. The
3525    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3526    restart_policy: RestartPolicy,
3527    effective_drain_timeout: Arc<Mutex<Duration>>,
3528    provenance_probe: ExecutableIdentityProbe,
3529}
3530
3531impl fmt::Debug for SupervisedModule {
3532    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3533        f.debug_struct("SupervisedModule")
3534            .field("module_id", &self.inner.module_id)
3535            .field("status", &self.status())
3536            .finish_non_exhaustive()
3537    }
3538}
3539
3540impl SupervisedModule {
3541    pub fn module_id(&self) -> &str {
3542        &self.inner.module_id
3543    }
3544
3545    /// Test-only: put one probe miss on the streak, the way
3546    /// `handle_health_probe_failure` does, so tests can assert what a later
3547    /// event does to the streak without driving the whole probe loop.
3548    #[cfg(test)]
3549    pub(crate) fn record_health_probe_failure_for_test(
3550        &self,
3551        detail: &str,
3552    ) -> Result<(), SuperviseError> {
3553        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3554            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3555            state.health.detail = Some(detail.to_string());
3556        })
3557    }
3558
3559    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3560        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3561    }
3562
3563    pub(crate) fn registration_end_reason(
3564        &self,
3565    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3566        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3567        Ok(match snapshot.state {
3568            ModuleState::Draining if snapshot.draining_to_replace => {
3569                Some(RegistrationEndReason::SupervisorRestart)
3570            }
3571            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3572            ModuleState::Restarting if snapshot.draining_to_replace => {
3573                Some(RegistrationEndReason::SupervisorRestart)
3574            }
3575            _ => None,
3576        })
3577    }
3578
3579    /// The module's retained stderr, newest lines last.
3580    ///
3581    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3582    /// module, `supervisor.list` renders every module, and putting it in the
3583    /// shared snapshot would make each status read carry a payload almost nobody
3584    /// asked for. Callers that want the text ask for it.
3585    pub fn stderr_tail(
3586        &self,
3587        max_lines: Option<usize>,
3588        max_bytes: Option<usize>,
3589    ) -> StderrTailSnapshot {
3590        self.inner
3591            .stderr_ring
3592            .lock()
3593            .unwrap_or_else(|poisoned| poisoned.into_inner())
3594            .snapshot(max_lines, max_bytes)
3595    }
3596
3597    /// The module's bounded terminal history, oldest retained exit first.
3598    ///
3599    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3600    /// daemon whose in-memory history was necessarily reset.
3601    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3602        self.inner
3603            .terminal_ring
3604            .lock()
3605            .unwrap_or_else(|poisoned| poisoned.into_inner())
3606            .snapshot()
3607    }
3608
3609    /// Retained observations from the current ring and all journal generations.
3610    ///
3611    /// Blocking: this reads the journal files. Async callers use
3612    /// [`Self::read_durable_terminal_history`].
3613    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3614        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3615    }
3616
3617    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3618    /// read (up to every retained generation) never occupies a runtime worker.
3619    /// Fails only if the blocking task could not finish (runtime shutdown or a
3620    /// panic in the read).
3621    pub(crate) async fn read_durable_terminal_history(
3622        &self,
3623    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3624        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3625        let module_id = self.inner.module_id.clone();
3626        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3627            .await
3628    }
3629
3630    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3631        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3632            .map(|(status, _)| status)
3633    }
3634
3635    pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3636        let state = lock_snapshot(&self.inner.snapshot)?;
3637        Ok(state
3638            .failed_retry
3639            .pending
3640            .filter(|_| state.state == ModuleState::Failed && state.enabled)
3641            .map(|retry| {
3642                unix_ms_now().saturating_add(
3643                    u64::try_from(
3644                        retry
3645                            .deadline
3646                            .saturating_duration_since(Instant::now())
3647                            .as_millis(),
3648                    )
3649                    .unwrap_or(u64::MAX),
3650                )
3651            }))
3652    }
3653
3654    pub(crate) fn record_deliberate_severance(
3655        &self,
3656        identity: ProcessIdentity,
3657    ) -> Result<bool, SuperviseError> {
3658        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3659        if snapshot.pid != Some(identity.pid)
3660            || snapshot.process_start_time != Some(identity.start_time)
3661        {
3662            return Ok(false);
3663        }
3664        snapshot.deliberate_severance = Some(identity);
3665        Ok(true)
3666    }
3667
3668    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3669    ///
3670    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3671    /// does not produce reader-observability logs.
3672    pub(crate) fn status_for_control(
3673        &self,
3674        caller: &'static str,
3675    ) -> Result<ModuleStatus, SuperviseError> {
3676        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3677            .map(|(status, _)| status)
3678    }
3679
3680    fn status_with_snapshot_lock(
3681        &self,
3682        snapshot: &SharedSnapshot,
3683        caller: Option<&'static str>,
3684    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3685        let mut guard = match caller {
3686            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3687            None => lock_snapshot(snapshot)?,
3688        };
3689        // Read the budget through the pruning path so a reader sees the same
3690        // in-window count the restart decision would use, not a stale total.
3691        let restart_count =
3692            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3693        let snapshot = guard.clone();
3694        drop(guard);
3695        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3696            SuperviseError::StatePoisoned {
3697                module_id: Some(self.inner.module_id.clone()),
3698            }
3699        })?;
3700        let registration_active = self
3701            .inner
3702            .registry
3703            .get_module(&self.inner.module_id)
3704            .map_err(SuperviseError::Registry)?
3705            .is_some();
3706        let protocol = snapshot
3707            .spawned_protocol
3708            .unwrap_or(self.declared_protocol()?);
3709        let running_process =
3710            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3711        // Registration is the difference between the two protocols and the only
3712        // one: a subc module that has not registered cannot serve a request even
3713        // though its process is up, and a `none` module never registers at all,
3714        // so requiring it there would pin `live` to false for the whole life of
3715        // a perfectly healthy process.
3716        let live = match protocol {
3717            ModuleProtocol::Subc => running_process && registration_active,
3718            ModuleProtocol::None => running_process,
3719        };
3720
3721        Ok((
3722            ModuleStatus {
3723                waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3724                waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3725                module_id: self.inner.module_id.clone(),
3726                launch_nonce_source: observed_launch_nonce_source(&snapshot),
3727                state: snapshot.state,
3728                enabled: snapshot.enabled,
3729                process_alive: snapshot.process_alive,
3730                registration_active,
3731                protocol,
3732                live,
3733                restart_count,
3734                lifetime_restarts: snapshot.lifetime_restarts,
3735                spawn_generation: snapshot.spawn_generation,
3736                max_restarts: self.inner.restart_policy.max_restarts,
3737                restart_window: self.inner.restart_policy.window,
3738                drain_timeout,
3739                restart_backoff: self.inner.restart_policy.backoff,
3740                restart_max_backoff: self.inner.restart_policy.max_backoff,
3741                pid: snapshot.reported_pid(),
3742                spawned_at_ms: snapshot.spawned_at_ms,
3743                spawned_from: snapshot.spawned_from,
3744                launch_spec_digest: snapshot.launch_spec_digest,
3745                process_start_time: snapshot.process_start_time,
3746                last_exit: snapshot.last_exit,
3747                health: snapshot.health,
3748            },
3749            snapshot.spawned_file_identity.clone(),
3750        ))
3751    }
3752
3753    #[cfg(test)]
3754    pub(crate) fn hold_snapshot_for_test(
3755        &self,
3756        acquired: std::sync::mpsc::Sender<()>,
3757        hold: Duration,
3758    ) -> std::thread::JoinHandle<()> {
3759        let snapshot = Arc::clone(&self.inner.snapshot);
3760        std::thread::spawn(move || {
3761            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3762            acquired
3763                .send(())
3764                .expect("test receiver waits for snapshot lock");
3765            std::thread::sleep(hold);
3766        })
3767    }
3768
3769    /// The status and the running-image check for `supervisor.provenance`,
3770    /// taken from one status read. The exec acknowledgement can land between
3771    /// two separate reads, and the reply would then pair "no pid yet" with an
3772    /// image observed after the module started, which describes no single
3773    /// moment.
3774    pub(crate) async fn status_and_running_image_agreement(
3775        &self,
3776    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3777        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3778        let image = self
3779            .inner
3780            .provenance_probe
3781            .observe(
3782                status.pid,
3783                status.spawned_from.as_deref(),
3784                identity,
3785                status.process_start_time,
3786            )
3787            .await;
3788        Ok((status, image))
3789    }
3790
3791    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3792        let snapshot = match lock_snapshot(&self.inner.snapshot) {
3793            Ok(snapshot) => snapshot.clone(),
3794            Err(_) => {
3795                return subc_control::RunningImageAgreement::Unavailable {
3796                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
3797                };
3798            }
3799        };
3800        self.inner
3801            .provenance_probe
3802            .observe(
3803                snapshot.reported_pid(),
3804                snapshot.spawned_from.as_deref(),
3805                snapshot.spawned_file_identity.clone(),
3806                snapshot.process_start_time,
3807            )
3808            .await
3809    }
3810
3811    /// Memory and CPU time of the module's current process, read now. Only the
3812    /// process the supervisor spawned is read, not processes it has started.
3813    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3814        #[cfg(windows)]
3815        {
3816            let image = lock_snapshot(&self.inner.snapshot)
3817                .ok()
3818                .and_then(|snapshot| {
3819                    snapshot
3820                        .spawned_file_identity
3821                        .as_ref()
3822                        .and_then(|identity| identity.windows_image.clone())
3823                });
3824            match image {
3825                Some(image) => {
3826                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
3827                }
3828                None => subc_control::ChildResourceUsage::Unavailable {
3829                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3830                },
3831            }
3832        }
3833        #[cfg(not(windows))]
3834        {
3835            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3836                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3837                Err(_) => {
3838                    return subc_control::ChildResourceUsage::Unavailable {
3839                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3840                    }
3841                }
3842            };
3843            crate::child_resources::read(pid, start_time)
3844        }
3845    }
3846
3847    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3848        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3849        Ok(match snapshot.state {
3850            ModuleState::Restarting => true,
3851            ModuleState::Failed | ModuleState::Disabled => false,
3852            _ if snapshot.enabled && snapshot.respawn_pending => true,
3853            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3854        })
3855    }
3856
3857    #[cfg(test)]
3858    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3859        self.is_warming_with_snapshot_lock(None)
3860    }
3861
3862    pub(crate) fn is_warming_for_control(
3863        &self,
3864        caller: &'static str,
3865    ) -> Result<bool, SuperviseError> {
3866        self.is_warming_with_snapshot_lock(Some(caller))
3867    }
3868
3869    fn is_warming_with_snapshot_lock(
3870        &self,
3871        caller: Option<&'static str>,
3872    ) -> Result<bool, SuperviseError> {
3873        let snapshot = match caller {
3874            Some(caller) => {
3875                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3876            }
3877            None => lock_snapshot(&self.inner.snapshot)?,
3878        }
3879        .clone();
3880        Ok(matches!(
3881            snapshot.state,
3882            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3883        ))
3884    }
3885
3886    /// Drain the module and stop monitoring it.
3887    pub async fn drain(&self) -> Result<(), SuperviseError> {
3888        self.stop().await
3889    }
3890
3891    pub(crate) async fn retire_requested(
3892        &self,
3893        requester: Option<Principal>,
3894    ) -> Result<(), SuperviseError> {
3895        match self.state()? {
3896            ModuleState::Stopped => return Ok(()),
3897            ModuleState::Failed
3898                if lock_snapshot(&self.inner.snapshot)?
3899                    .failed_retry
3900                    .pending
3901                    .is_none() =>
3902            {
3903                return Ok(())
3904            }
3905            ModuleState::Failed
3906            | ModuleState::Starting
3907            | ModuleState::Running
3908            | ModuleState::Unresponsive
3909            | ModuleState::Restarting
3910            | ModuleState::Draining
3911            | ModuleState::Disabled => {}
3912        }
3913
3914        let (reply_tx, reply_rx) = oneshot::channel();
3915        self.inner
3916            .commands
3917            .send(SupervisorCommand::Retire {
3918                requester,
3919                reply: reply_tx,
3920            })
3921            .await
3922            .map_err(|_| SuperviseError::CommandClosed {
3923                module_id: self.inner.module_id.clone(),
3924            })?;
3925        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3926            module_id: self.inner.module_id.clone(),
3927        })?
3928    }
3929
3930    pub async fn stop(&self) -> Result<(), SuperviseError> {
3931        self.stop_requested(None).await
3932    }
3933
3934    pub(crate) async fn stop_requested(
3935        &self,
3936        requester: Option<Principal>,
3937    ) -> Result<(), SuperviseError> {
3938        match self.state()? {
3939            ModuleState::Stopped => return Ok(()),
3940            ModuleState::Failed
3941                if lock_snapshot(&self.inner.snapshot)?
3942                    .failed_retry
3943                    .pending
3944                    .is_none() =>
3945            {
3946                return Ok(())
3947            }
3948            ModuleState::Failed
3949            | ModuleState::Starting
3950            | ModuleState::Running
3951            | ModuleState::Unresponsive
3952            | ModuleState::Restarting
3953            | ModuleState::Draining
3954            | ModuleState::Disabled => {}
3955        }
3956
3957        let (reply_tx, reply_rx) = oneshot::channel();
3958        self.inner
3959            .commands
3960            .send(SupervisorCommand::Drain {
3961                requester,
3962                reply: reply_tx,
3963            })
3964            .await
3965            .map_err(|_| SuperviseError::CommandClosed {
3966                module_id: self.inner.module_id.clone(),
3967            })?;
3968        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3969            module_id: self.inner.module_id.clone(),
3970        })?
3971    }
3972
3973    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3974        self.restart_requested(drain_timeout_ms, None).await
3975    }
3976
3977    pub(crate) async fn restart_requested(
3978        &self,
3979        drain_timeout_ms: Option<u64>,
3980        requester: Option<Principal>,
3981    ) -> Result<(), SuperviseError> {
3982        let queued_at = Instant::now();
3983        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3984        let (reply_tx, reply_rx) = oneshot::channel();
3985        self.inner
3986            .commands
3987            .send(SupervisorCommand::Restart {
3988                requester,
3989                drain_timeout_ms,
3990                received_at_generation,
3991                queued_at,
3992                reply: reply_tx,
3993            })
3994            .await
3995            .map_err(|_| SuperviseError::CommandClosed {
3996                module_id: self.inner.module_id.clone(),
3997            })?;
3998        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3999            module_id: self.inner.module_id.clone(),
4000        })?
4001    }
4002
4003    /// Blue/green restart: see [`SupervisorCommand::Swap`] and the
4004    /// `supervisor_swap` module. Returns once the swap has cut over (the old
4005    /// process then drains in the background of the supervise loop) or has
4006    /// failed, leaving the old process serving.
4007    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4008        self.swap_requested(ready_timeout, None).await
4009    }
4010
4011    pub(crate) async fn swap_requested(
4012        &self,
4013        ready_timeout: Option<Duration>,
4014        requester: Option<Principal>,
4015    ) -> Result<(), SuperviseError> {
4016        let (reply_tx, reply_rx) = oneshot::channel();
4017        self.inner
4018            .commands
4019            .send(SupervisorCommand::Swap {
4020                requester,
4021                ready_timeout,
4022                reply: reply_tx,
4023            })
4024            .await
4025            .map_err(|_| SuperviseError::CommandClosed {
4026                module_id: self.inner.module_id.clone(),
4027            })?;
4028        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4029            module_id: self.inner.module_id.clone(),
4030        })?
4031    }
4032
4033    pub async fn reload(&self) -> Result<(), SuperviseError> {
4034        self.reload_requested(None).await
4035    }
4036
4037    pub(crate) async fn reload_requested(
4038        &self,
4039        requester: Option<Principal>,
4040    ) -> Result<(), SuperviseError> {
4041        let (reply_tx, reply_rx) = oneshot::channel();
4042        self.inner
4043            .commands
4044            .send(SupervisorCommand::Reload {
4045                requester,
4046                reply: reply_tx,
4047            })
4048            .await
4049            .map_err(|_| SuperviseError::CommandClosed {
4050                module_id: self.inner.module_id.clone(),
4051            })?;
4052        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4053            module_id: self.inner.module_id.clone(),
4054        })?
4055    }
4056
4057    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4058        self.set_enabled_requested(enabled, None).await
4059    }
4060
4061    pub(crate) async fn set_enabled_requested(
4062        &self,
4063        enabled: bool,
4064        requester: Option<Principal>,
4065    ) -> Result<bool, SuperviseError> {
4066        let (reply_tx, reply_rx) = oneshot::channel();
4067        self.inner
4068            .commands
4069            .send(SupervisorCommand::SetEnabled {
4070                requester,
4071                enabled,
4072                reply: reply_tx,
4073            })
4074            .await
4075            .map_err(|_| SuperviseError::CommandClosed {
4076                module_id: self.inner.module_id.clone(),
4077            })?;
4078        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4079            module_id: self.inner.module_id.clone(),
4080        })?
4081    }
4082
4083    /// The current process's protocol, or the configured protocol when down.
4084    /// A rescan stores the next launch spec without changing how an existing
4085    /// process registers, serves routes, is probed, or exits.
4086    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4087        let configured = self
4088            .inner
4089            .configuration
4090            .lock()
4091            .map_err(|_| SuperviseError::StatePoisoned {
4092                module_id: Some(self.inner.module_id.clone()),
4093            })?
4094            .spec
4095            .protocol;
4096        let state = lock_snapshot(&self.inner.snapshot)?;
4097        Ok(state.spawned_protocol.unwrap_or(configured))
4098    }
4099
4100    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4101        let configuration =
4102            self.inner
4103                .configuration
4104                .lock()
4105                .map_err(|_| SuperviseError::StatePoisoned {
4106                    module_id: Some(self.inner.module_id.clone()),
4107                })?;
4108        Ok((configuration.spec.clone(), configuration.health.clone()))
4109    }
4110
4111    /// Replace this module's launch spec, keeping its health and drain policy,
4112    /// the way a rescan does for a changed config entry. The running process is
4113    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
4114    #[cfg(any(test, feature = "test-support"))]
4115    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4116        let (_, health) = self.configuration()?;
4117        let drain_timeout_ms = u64::try_from(
4118            self.inner
4119                .effective_drain_timeout
4120                .lock()
4121                .unwrap_or_else(|poisoned| poisoned.into_inner())
4122                .as_millis(),
4123        )
4124        .ok();
4125        self.update_configuration(spec, health, drain_timeout_ms)
4126            .await
4127    }
4128
4129    pub(crate) async fn update_configuration(
4130        &self,
4131        spec: ModuleSpec,
4132        health: HealthConfig,
4133        drain_timeout_ms: Option<u64>,
4134    ) -> Result<(), SuperviseError> {
4135        if spec.module_id != self.inner.module_id {
4136            return Err(SuperviseError::InvalidSpec {
4137                reason: "a supervised module's module_id cannot be changed".to_string(),
4138            });
4139        }
4140        validate_spec(&spec)?;
4141        let (reply_tx, reply_rx) = oneshot::channel();
4142        self.inner
4143            .commands
4144            .send(SupervisorCommand::UpdateConfiguration {
4145                spec: spec.clone(),
4146                health: health.clone(),
4147                drain_timeout_ms,
4148                reply: reply_tx,
4149            })
4150            .await
4151            .map_err(|_| SuperviseError::CommandClosed {
4152                module_id: self.inner.module_id.clone(),
4153            })?;
4154        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4155            module_id: self.inner.module_id.clone(),
4156        })?;
4157        let mut configuration =
4158            self.inner
4159                .configuration
4160                .lock()
4161                .map_err(|_| SuperviseError::StatePoisoned {
4162                    module_id: Some(self.inner.module_id.clone()),
4163                })?;
4164        configuration.spec = spec;
4165        configuration.health = health;
4166        Ok(())
4167    }
4168}
4169
4170impl Drop for SupervisedModuleInner {
4171    fn drop(&mut self) {
4172        let Ok(mut monitor) = self.monitor.lock() else {
4173            return;
4174        };
4175        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4176            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4177                state.state = ModuleState::Stopped;
4178                clear_current_process_facts(state);
4179            });
4180            monitor.abort();
4181        }
4182        let _ = monitor.take();
4183    }
4184}
4185
4186#[derive(Debug)]
4187enum SupervisorCommand {
4188    Drain {
4189        requester: Option<Principal>,
4190        reply: oneshot::Sender<Result<(), SuperviseError>>,
4191    },
4192    Retire {
4193        requester: Option<Principal>,
4194        reply: oneshot::Sender<Result<(), SuperviseError>>,
4195    },
4196    Restart {
4197        requester: Option<Principal>,
4198        /// Operator override for this one restart's drain budget, in ms. `None`
4199        /// uses the module's configured/default budget; `Some(0)` cuts
4200        /// immediately (wedge bounce: a stuck request never settles, so
4201        /// waiting only delays recovery).
4202        drain_timeout_ms: Option<u64>,
4203        /// The module's `spawn_generation` when the request was received, before
4204        /// it waited in the command queue. A queued restart whose module has
4205        /// since spawned a newer process is already satisfied (see the handler).
4206        received_at_generation: u64,
4207        /// When the request entered the command queue, so the handler can log
4208        /// how long it waited behind the loop's other work.
4209        queued_at: Instant,
4210        reply: oneshot::Sender<Result<(), SuperviseError>>,
4211    },
4212    Reload {
4213        requester: Option<Principal>,
4214        reply: oneshot::Sender<Result<(), SuperviseError>>,
4215    },
4216    SetEnabled {
4217        requester: Option<Principal>,
4218        enabled: bool,
4219        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4220    },
4221    UpdateConfiguration {
4222        spec: ModuleSpec,
4223        health: HealthConfig,
4224        /// Per-module drain override from the new config; `None` re-resolves to
4225        /// the supervisor-wide default.
4226        drain_timeout_ms: Option<u64>,
4227        reply: oneshot::Sender<()>,
4228    },
4229    Swap {
4230        requester: Option<Principal>,
4231        /// How long the candidate may take to register and declare itself
4232        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4233        ready_timeout: Option<Duration>,
4234        /// Answered at cutover or failure; the incumbent's drain follows.
4235        reply: oneshot::Sender<Result<(), SuperviseError>>,
4236    },
4237}
4238
4239#[derive(Debug)]
4240pub enum SuperviseError {
4241    InvalidSpec {
4242        reason: String,
4243    },
4244    Spawn {
4245        program: PathBuf,
4246        source: io::Error,
4247        cgroup_path: Option<PathBuf>,
4248    },
4249    Cgroup {
4250        module_id: String,
4251        source: io::Error,
4252    },
4253    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4254    /// than spawn a reserved module without its identity binding.
4255    LaunchNonce {
4256        reason: String,
4257    },
4258    Wait {
4259        module_id: String,
4260        source: io::Error,
4261    },
4262    Kill {
4263        module_id: String,
4264        source: io::Error,
4265    },
4266    Forwarding(ForwardingError),
4267    Registry(RegistryError),
4268    ReloadUnavailable {
4269        module_id: String,
4270        reason: String,
4271    },
4272    /// An operator restart/reload was requested for a module that is currently
4273    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4274    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4275    /// by a restart, so these commands are rejected instead of re-enabling it.
4276    Disabled {
4277        module_id: String,
4278    },
4279    ReloadFailed {
4280        module_id: String,
4281        reason: String,
4282    },
4283    RegistrationStillActive {
4284        module_id: String,
4285        waited: Duration,
4286    },
4287    StatePoisoned {
4288        module_id: Option<String>,
4289    },
4290    CommandClosed {
4291        module_id: String,
4292    },
4293    /// A restart or reload arrived while a swap's candidate was warming. The
4294    /// swap owns the module until it cuts over or fails; a stop or disable
4295    /// would have aborted it instead.
4296    SwapInProgress {
4297        module_id: String,
4298    },
4299    /// A swap was refused before anything was spawned.
4300    SwapRefused {
4301        module_id: String,
4302        reason: SwapRefusal,
4303    },
4304    /// A swap spawned a candidate and gave up on it. The candidate has been
4305    /// killed and its slot freed; the incumbent was left serving and was never
4306    /// drained, except in the one `CutoverLost` case described on that arm.
4307    SwapFailed {
4308        module_id: String,
4309        arm: SwapFailureArm,
4310        detail: String,
4311        /// How the candidate exited, when it exited on its own before the
4312        /// supervisor gave up on it.
4313        candidate_exit: Option<ExitReport>,
4314    },
4315}
4316
4317/// Why a swap was refused before a candidate was spawned.
4318#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4319pub enum SwapRefusal {
4320    /// The module's config does not declare `overlap: "safe"`.
4321    OverlapExclusive,
4322    /// The module is not registered, so there is no incumbent to keep serving
4323    /// and nothing a swap would improve on; a plain restart is the tool.
4324    NotRegistered,
4325    /// The module does not speak the subc wire, so a candidate could never
4326    /// register or declare itself ready.
4327    ProtocolNone,
4328    /// The supervisor lacks the forwarding table (to cut routes over) or the
4329    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4330    NotConfigured,
4331    /// A swap is already open for this module.
4332    AlreadySwapping,
4333}
4334
4335impl SwapRefusal {
4336    pub fn as_str(self) -> &'static str {
4337        match self {
4338            Self::OverlapExclusive => "overlap_exclusive",
4339            Self::NotRegistered => "not_registered",
4340            Self::ProtocolNone => "protocol_none",
4341            Self::NotConfigured => "not_configured",
4342            Self::AlreadySwapping => "already_swapping",
4343        }
4344    }
4345}
4346
4347/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4348/// serving and undrained; see `CutoverLost`.
4349#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4350pub enum SwapFailureArm {
4351    /// The candidate process could not be started.
4352    SpawnFailed,
4353    /// The candidate did not register within the readiness budget.
4354    NeverRegistered,
4355    /// The candidate registered but did not declare itself ready in time.
4356    NeverReady,
4357    /// The candidate exited before cutover.
4358    CandidateExited,
4359    /// The candidate declared itself ready but failed its health probe.
4360    CandidateUnhealthy,
4361    /// An operator stop, disable or retire arrived while the candidate warmed.
4362    /// The candidate was killed and the operator's command then carried out on
4363    /// the incumbent.
4364    Interrupted,
4365    /// The candidate's connection closed at the moment of cutover. If it
4366    /// closed before forwarding moved, the incumbent is untouched. If it closed
4367    /// between the forwarding and registry halves of cutover, forwarding can no
4368    /// longer route to the incumbent, so the module is restarted plainly.
4369    CutoverLost,
4370}
4371
4372impl SwapFailureArm {
4373    pub fn as_str(self) -> &'static str {
4374        match self {
4375            Self::SpawnFailed => "spawn_failed",
4376            Self::NeverRegistered => "never_registered",
4377            Self::NeverReady => "never_ready",
4378            Self::CandidateExited => "candidate_exited",
4379            Self::CandidateUnhealthy => "candidate_unhealthy",
4380            Self::Interrupted => "interrupted",
4381            Self::CutoverLost => "cutover_lost",
4382        }
4383    }
4384}
4385
4386impl fmt::Display for SuperviseError {
4387    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4388        match self {
4389            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4390            Self::Spawn {
4391                program,
4392                source,
4393                cgroup_path: Some(cgroup_path),
4394            } => write!(
4395                f,
4396                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4397                cgroup_path.display(),
4398                program.display()
4399            ),
4400            Self::Spawn {
4401                program,
4402                source,
4403                cgroup_path: None,
4404            } => write!(
4405                f,
4406                "failed to spawn module '{}': {source}",
4407                program.display()
4408            ),
4409            Self::Cgroup { module_id, source } => {
4410                write!(
4411                    f,
4412                    "failed to prepare cgroup for module '{module_id}': {source}"
4413                )
4414            }
4415            Self::LaunchNonce { reason } => {
4416                write!(
4417                    f,
4418                    "failed to generate reserved-module launch nonce: {reason}"
4419                )
4420            }
4421            Self::Wait { module_id, source } => {
4422                write!(f, "failed to wait for module '{module_id}': {source}")
4423            }
4424            Self::Kill { module_id, source } => {
4425                write!(f, "failed to kill module '{module_id}': {source}")
4426            }
4427            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4428            Self::Registry(err) => write!(f, "registry error: {err}"),
4429            Self::ReloadUnavailable { module_id, reason } => {
4430                write!(f, "reload unavailable for module '{module_id}': {reason}")
4431            }
4432            Self::Disabled { module_id } => {
4433                write!(
4434                    f,
4435                    "module '{module_id}' is disabled; enable it before restart or reload"
4436                )
4437            }
4438            Self::ReloadFailed { module_id, reason } => {
4439                write!(f, "reload failed for module '{module_id}': {reason}")
4440            }
4441            Self::RegistrationStillActive { module_id, waited } => write!(
4442                f,
4443                "module '{module_id}' registration remained active after waiting {waited:?}"
4444            ),
4445            Self::StatePoisoned { module_id } => match module_id {
4446                Some(module_id) => {
4447                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4448                }
4449                None => write!(f, "supervisor state was poisoned"),
4450            },
4451            Self::CommandClosed { module_id } => {
4452                write!(
4453                    f,
4454                    "supervisor command channel for module '{module_id}' is closed"
4455                )
4456            }
4457            Self::SwapInProgress { module_id } => write!(
4458                f,
4459                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4460            ),
4461            Self::SwapRefused { module_id, reason } => match reason {
4462                SwapRefusal::OverlapExclusive => write!(
4463                    f,
4464                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4465                ),
4466                SwapRefusal::NotRegistered => write!(
4467                    f,
4468                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4469                ),
4470                SwapRefusal::ProtocolNone => write!(
4471                    f,
4472                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4473                ),
4474                SwapRefusal::NotConfigured => write!(
4475                    f,
4476                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4477                ),
4478                SwapRefusal::AlreadySwapping => {
4479                    write!(f, "module '{module_id}' is already being swapped")
4480                }
4481            },
4482            Self::SwapFailed {
4483                module_id,
4484                arm,
4485                detail,
4486                ..
4487            } => write!(
4488                f,
4489                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4490                arm.as_str()
4491            ),
4492        }
4493    }
4494}
4495
4496impl Error for SuperviseError {
4497    fn source(&self) -> Option<&(dyn Error + 'static)> {
4498        match self {
4499            Self::Spawn { source, .. }
4500            | Self::Cgroup { source, .. }
4501            | Self::Wait { source, .. }
4502            | Self::Kill { source, .. } => Some(source),
4503            Self::Forwarding(err) => Some(err),
4504            Self::Registry(err) => Some(err),
4505            Self::LaunchNonce { .. }
4506            | Self::InvalidSpec { .. }
4507            | Self::ReloadUnavailable { .. }
4508            | Self::Disabled { .. }
4509            | Self::ReloadFailed { .. }
4510            | Self::RegistrationStillActive { .. }
4511            | Self::StatePoisoned { .. }
4512            | Self::CommandClosed { .. }
4513            | Self::SwapInProgress { .. }
4514            | Self::SwapRefused { .. }
4515            | Self::SwapFailed { .. } => None,
4516        }
4517    }
4518}
4519
4520pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4521    if spec.module_id.trim().is_empty() {
4522        return Err(SuperviseError::InvalidSpec {
4523            reason: "module_id must not be empty".to_string(),
4524        });
4525    }
4526
4527    Ok(())
4528}
4529
4530#[derive(Debug, Default)]
4531struct HealthProbeRuntime {
4532    spawn_generation: Option<u64>,
4533    configured_health: Option<HealthConfig>,
4534    registered_connection: Option<crate::ConnectionId>,
4535    advertised: bool,
4536    next_probe_at: Option<Instant>,
4537    probe_index: u64,
4538}
4539
4540fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4541    lock_snapshot(snapshot)
4542        .ok()
4543        .and_then(|state| state.spawned_protocol)
4544        .unwrap_or(spec.protocol)
4545}
4546
4547impl HealthProbeRuntime {
4548    fn refresh_registration(
4549        &mut self,
4550        spec: &ModuleSpec,
4551        runtime: &SupervisorRuntimeConfig,
4552        registry: &Registry,
4553        snapshot: &SharedSnapshot,
4554    ) {
4555        let generation = lock_snapshot(snapshot)
4556            .ok()
4557            .map(|state| state.spawn_generation);
4558        if self.spawn_generation != generation {
4559            // A replacement cannot reuse a deadline that is already due for its
4560            // predecessor. HTTP children get a full cadence to start; wire
4561            // children still wait for their own health-advertising HELLO.
4562            *self = Self {
4563                spawn_generation: generation,
4564                ..Self::default()
4565            };
4566        }
4567        if self.configured_health.as_ref() != Some(&runtime.health) {
4568            self.configured_health = Some(runtime.health.clone());
4569            self.next_probe_at = None;
4570            self.registered_connection = None;
4571            self.probe_index = 0;
4572        }
4573        // A non-wire process never registers. Only an explicitly configured
4574        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4575        // health failure for that kind of process.
4576        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4577            self.registered_connection = None;
4578            self.advertised = runtime.health.http.is_some();
4579            if !self.advertised {
4580                self.next_probe_at = None;
4581                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4582                    let unknown = ModuleHealthStatus::default();
4583                    if state.health != unknown {
4584                        state.health = unknown;
4585                    }
4586                });
4587            } else if self.next_probe_at.is_none() {
4588                self.next_probe_at = Some(
4589                    Instant::now()
4590                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4591                );
4592            }
4593            return;
4594        }
4595
4596        let registration = match registry.get_module(&spec.module_id) {
4597            Ok(registration) => registration,
4598            Err(err) => {
4599                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4600                self.advertised = false;
4601                self.next_probe_at = None;
4602                return;
4603            }
4604        };
4605
4606        let Some(registration) = registration else {
4607            self.registered_connection = None;
4608            self.advertised = false;
4609            self.next_probe_at = None;
4610            return;
4611        };
4612
4613        let advertised = registration
4614            .control_ops
4615            .iter()
4616            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4617        if !advertised {
4618            self.registered_connection = Some(registration.connection_id);
4619            self.advertised = false;
4620            self.next_probe_at = None;
4621            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4622                if state.health.status != SupervisorHealthStatus::Unknown
4623                    || state.health.consecutive_failures != 0
4624                    || state.health.last_probe_ms.is_some()
4625                    || state.health.detail.is_some()
4626                    || state.health.metrics.is_some()
4627                {
4628                    state.health.status = SupervisorHealthStatus::Unknown;
4629                    state.health.consecutive_failures = 0;
4630                    state.health.last_probe_ms = None;
4631                    state.health.detail = None;
4632                    state.health.metrics = None;
4633                }
4634            });
4635            return;
4636        }
4637
4638        let reregistered = self.registered_connection != Some(registration.connection_id);
4639        self.registered_connection = Some(registration.connection_id);
4640        self.advertised = true;
4641        if reregistered || self.next_probe_at.is_none() {
4642            self.probe_index = 0;
4643            self.next_probe_at = Some(
4644                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4645            );
4646            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4647                state.health.status = SupervisorHealthStatus::Unknown;
4648                state.health.consecutive_failures = 0;
4649                state.health.detail = None;
4650                state.health.metrics = None;
4651            });
4652        }
4653    }
4654
4655    fn wake_at(&self) -> Option<Instant> {
4656        if !self.advertised {
4657            return None;
4658        }
4659        self.next_probe_at
4660    }
4661
4662    fn due(&self) -> bool {
4663        self.advertised
4664            && self
4665                .next_probe_at
4666                .is_some_and(|next| Instant::now() >= next)
4667    }
4668
4669    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4670        self.probe_index = self.probe_index.wrapping_add(1);
4671        self.next_probe_at = Some(
4672            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4673        );
4674    }
4675}
4676
4677/// What a failed health probe actually OBSERVED, kept apart from how it reads.
4678///
4679/// This was a struct with a single `message: String`, and every one of the
4680/// fifteen construction sites collapsed into it. Each site knows exactly what it
4681/// saw -- the lane is gone, the module did not answer in time, the module
4682/// answered with the wrong thing -- and `handle_health_probe_failure` then
4683/// treated all of them identically: increment a counter, compare to a threshold,
4684/// restart the module. THE DISTINCTION EXISTED AT EVERY CALL SITE AND WAS
4685/// DESTROYED BEFORE THE DECISION THAT NEEDED IT.
4686///
4687/// The distinction that matters is not severity, it is EVIDENTIAL WEIGHT:
4688///
4689/// * `LaneDead` is PROOF. The module's control connection is gone; nothing will
4690///   answer on it again.
4691/// * `NoAnswer` is ABSENCE OF EVIDENCE. It is consistent with a wedged module
4692///   AND with a perfectly healthy one that lost a CPU race -- which is what
4693///   happens under machine load, and is how this supervisor killed a healthy
4694///   module three times in one day.
4695/// * `BadAnswer` proves the module is ALIVE. It replied; the reply was wrong.
4696///   Restarting on it is defensible, but it is not the silence case and should
4697///   never be counted as one.
4698/// * `Misconfigured` is a daemon-side fault. The module has not been asked
4699///   anything, so it cannot be evidence about the module at all.
4700///
4701/// The asymmetry is the whole point: under saturation the WEAKEST signal is the
4702/// one that fires most often, and while every variant collapsed into one string
4703/// it carried the same weight as the strongest.
4704///
4705/// LIVE BEHAVIOUR TODAY, stated here because this doc block describes the
4706/// DESIGN and a reader stopping at it gets the build backwards: the restart
4707/// decision does NOT yet consult this classification -- consecutive `NoAnswer`
4708/// probes still increment the failure streak and drive escalation at the
4709/// threshold (see `is_proof_of_death` below for why that is deliberate and
4710/// what gates the change). Absence of evidence restarts modules today.
4711#[derive(Debug)]
4712enum HealthProbeEvidence {
4713    /// The module's control lane is gone. Proof of death.
4714    LaneDead,
4715    /// No reply within the deadline. Proves nothing about the module's state.
4716    NoAnswer,
4717    /// The module replied, but not with a usable health report. Proves it is alive.
4718    BadAnswer,
4719    /// The daemon could not ask. Says nothing about the module.
4720    Misconfigured,
4721}
4722
4723#[derive(Debug)]
4724struct HealthProbeError {
4725    evidence: HealthProbeEvidence,
4726    message: String,
4727}
4728
4729impl HealthProbeError {
4730    fn lane_dead(message: impl Into<String>) -> Self {
4731        Self::with(HealthProbeEvidence::LaneDead, message)
4732    }
4733
4734    fn no_answer(message: impl Into<String>) -> Self {
4735        Self::with(HealthProbeEvidence::NoAnswer, message)
4736    }
4737
4738    fn bad_answer(message: impl Into<String>) -> Self {
4739        Self::with(HealthProbeEvidence::BadAnswer, message)
4740    }
4741
4742    fn misconfigured(message: impl Into<String>) -> Self {
4743        Self::with(HealthProbeEvidence::Misconfigured, message)
4744    }
4745
4746    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4747        Self {
4748            evidence,
4749            message: message.into(),
4750        }
4751    }
4752
4753    /// Whether this observation is proof the module cannot serve.
4754    ///
4755    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4756    /// variant that fires under CPU starvation, and treating it as proof is the
4757    /// defect this enum exists to make impossible to reintroduce silently.
4758    ///
4759    /// NOT YET CONSULTED BY THE RESTART DECISION, deliberately. Requiring proof
4760    /// to restart also needs a bound for the case it excludes -- a genuinely
4761    /// wedged module, alive but never answering -- and that bound must come from
4762    /// the distribution of real late-answer latencies, which nothing measures
4763    /// yet. Landing the classification first makes the later change a one-line
4764    /// decision against evidence that already exists, rather than two unproven
4765    /// changes at once.
4766    #[allow(dead_code)]
4767    fn is_proof_of_death(&self) -> bool {
4768        matches!(self.evidence, HealthProbeEvidence::LaneDead)
4769    }
4770
4771    /// Short stable label for logs and the health snapshot.
4772    ///
4773    /// An operator reading `ck health` currently cannot tell "the module is gone"
4774    /// from "the module did not answer in five seconds", because both render as
4775    /// prose in the same field. These labels are what make the two
4776    /// distinguishable at a glance, and they are what a later restart-policy
4777    /// change will be argued from.
4778    fn label(&self) -> &'static str {
4779        match self.evidence {
4780            HealthProbeEvidence::LaneDead => "lane-dead",
4781            HealthProbeEvidence::NoAnswer => "no-answer",
4782            HealthProbeEvidence::BadAnswer => "bad-answer",
4783            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4784        }
4785    }
4786}
4787
4788impl fmt::Display for HealthProbeError {
4789    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4790        f.write_str(&self.message)
4791    }
4792}
4793
4794async fn run_health_probe_cycle(
4795    spec: &ModuleSpec,
4796    runtime: &SupervisorRuntimeConfig,
4797    registry: &Registry,
4798    process_liveness: &SupervisorProcessLiveness,
4799    snapshot: &SharedSnapshot,
4800    child: &mut Option<SupervisedChild>,
4801) {
4802    let now_ms = unix_ms_now();
4803    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4804        .then_some(runtime.health.http.as_deref())
4805        .flatten();
4806    let result = match http {
4807        Some(url) => probe_http_health(url, runtime.health.deadline).await,
4808        None => probe_module_health(&spec.module_id, runtime, None).await,
4809    };
4810    match result {
4811        Ok(report) => {
4812            handle_health_report(
4813                spec,
4814                runtime,
4815                registry,
4816                process_liveness,
4817                snapshot,
4818                child,
4819                report,
4820                now_ms,
4821            )
4822            .await;
4823        }
4824        Err(err) => {
4825            if http.is_some() {
4826                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4827                    state.health.status = SupervisorHealthStatus::Failing;
4828                });
4829            }
4830            handle_health_probe_failure(
4831                spec,
4832                runtime,
4833                registry,
4834                process_liveness,
4835                snapshot,
4836                child,
4837                err,
4838                now_ms,
4839            )
4840            .await;
4841        }
4842    }
4843}
4844
4845pub(crate) struct HttpProbeTarget<'a> {
4846    address: std::net::SocketAddr,
4847    localhost: bool,
4848    authority: &'a str,
4849    path: String,
4850}
4851
4852/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
4853/// or TLS. A URL cannot turn a local health check into an outbound connection.
4854pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4855    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4856        return Err("must not contain whitespace, controls, or a fragment".into());
4857    }
4858    let rest = url
4859        .strip_prefix("http://")
4860        .ok_or("must use plain http://")?;
4861    let split = rest.find(['/', '?']).unwrap_or(rest.len());
4862    let (authority, suffix) = rest.split_at(split);
4863    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4864        ("::1", rest)
4865    } else {
4866        let split = authority.find(':').unwrap_or(authority.len());
4867        authority.split_at(split)
4868    };
4869    let ip: std::net::IpAddr = match host {
4870        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4871        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4872        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4873    };
4874    let port = if port.is_empty() {
4875        80
4876    } else {
4877        port.strip_prefix(':')
4878            .and_then(|p| p.parse::<u16>().ok())
4879            .filter(|p| *p > 0)
4880            .ok_or("must have a valid nonzero TCP port")?
4881    };
4882    let path = if suffix.is_empty() {
4883        "/".into()
4884    } else if suffix.starts_with('?') {
4885        format!("/{suffix}")
4886    } else {
4887        suffix.into()
4888    };
4889    Ok(HttpProbeTarget {
4890        address: std::net::SocketAddr::new(ip, port),
4891        localhost: host == "localhost",
4892        authority,
4893        path,
4894    })
4895}
4896
4897async fn probe_http_health(
4898    url: &str,
4899    deadline: Duration,
4900) -> Result<HealthReport, HealthProbeError> {
4901    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4902    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4903    // Keep partial diagnostics outside the timed future so cancellation does
4904    // not discard a status line or body bytes already received.
4905    let mut response_status = String::new();
4906    let mut body = Vec::new();
4907    let probe = async {
4908        // Resolve localhost ourselves so a hosts-file override cannot turn
4909        // this into an outbound request, while IPv6-only local servers work.
4910        let connection = match tokio::net::TcpStream::connect(target.address).await {
4911            Err(_) if target.localhost => {
4912                tokio::net::TcpStream::connect((
4913                    std::net::Ipv6Addr::LOCALHOST,
4914                    target.address.port(),
4915                ))
4916                .await
4917            }
4918            result => result,
4919        };
4920        let mut stream = connection.map_err(|error| {
4921            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4922        })?;
4923        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4924            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4925        let mut reader = BufReader::new(stream);
4926        let mut budget = 16 * 1024;
4927        let status = http_line(&mut reader, &mut budget).await?;
4928        let mut words = status.split_ascii_whitespace();
4929        let version = words.next();
4930        let code = words
4931            .next()
4932            .filter(|word| word.len() == 3)
4933            .and_then(|word| word.parse::<u16>().ok());
4934        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4935            || !code.is_some_and(|code| (100..600).contains(&code))
4936        {
4937            return Err(HealthProbeError::bad_answer(format!(
4938                "invalid HTTP status: {status}"
4939            )));
4940        }
4941        let code = code.expect("validated status code");
4942        response_status = status.clone();
4943        let mut length = None;
4944        let mut chunked = false;
4945        loop {
4946            let line = http_line(&mut reader, &mut budget).await?;
4947            if line.is_empty() {
4948                break;
4949            }
4950            if let Some((name, value)) = line.split_once(':') {
4951                if name.eq_ignore_ascii_case("content-length") {
4952                    length = Some(value.trim().parse::<u64>().map_err(|_| {
4953                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
4954                    })?);
4955                } else if name.eq_ignore_ascii_case("transfer-encoding") {
4956                    chunked = value.trim().eq_ignore_ascii_case("chunked");
4957                }
4958            }
4959        }
4960        if chunked {
4961            while body.len() < 200 {
4962                let line = http_line(&mut reader, &mut budget).await?;
4963                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4964                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4965                if size == 0 {
4966                    break;
4967                }
4968                let count = size.min((200 - body.len()) as u64) as usize;
4969                let start = body.len();
4970                (&mut reader)
4971                    .take(count as u64)
4972                    .read_to_end(&mut body)
4973                    .await
4974                    .map_err(|error| {
4975                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4976                    })?;
4977                if body.len() - start != count {
4978                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4979                }
4980                if size > count as u64 || body.len() == 200 {
4981                    break;
4982                }
4983                if !http_line(&mut reader, &mut budget).await?.is_empty() {
4984                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4985                }
4986            }
4987        } else {
4988            reader
4989                .take(length.unwrap_or(200).min(200))
4990                .read_to_end(&mut body)
4991                .await
4992                .map_err(|error| {
4993                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4994                })?;
4995        }
4996        if (200..300).contains(&code) {
4997            Ok(HealthReport::ok())
4998        } else {
4999            Err(HealthProbeError::bad_answer(
5000                "HTTP health endpoint returned non-2xx",
5001            ))
5002        }
5003    };
5004    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5005        Err(HealthProbeError::no_answer(format!(
5006            "HTTP probe timed out after {deadline:?}"
5007        )))
5008    });
5009    if let Err(error) = &mut result {
5010        if !response_status.is_empty() {
5011            error.message = format!(
5012                "{}; {response_status}: {}",
5013                error.message,
5014                String::from_utf8_lossy(&body)
5015            );
5016        }
5017    }
5018    result
5019}
5020
5021async fn http_line(
5022    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5023    remaining: &mut usize,
5024) -> Result<String, HealthProbeError> {
5025    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5026    let mut line = Vec::new();
5027    (&mut *reader)
5028        .take(*remaining as u64)
5029        .read_until(b'\n', &mut line)
5030        .await
5031        .map_err(|error| {
5032            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5033        })?;
5034    *remaining -= line.len();
5035    if !line.ends_with(b"\r\n") {
5036        return Err(HealthProbeError::bad_answer(
5037            "HTTP headers are incomplete or exceed 16 KiB",
5038        ));
5039    }
5040    line.truncate(line.len() - 2);
5041    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5042}
5043
5044async fn probe_module_health(
5045    module_id: &str,
5046    runtime: &SupervisorRuntimeConfig,
5047    drain_deadline: Option<Instant>,
5048) -> Result<HealthReport, HealthProbeError> {
5049    let Some(forwarding) = runtime.forwarding.as_ref() else {
5050        return Err(HealthProbeError::misconfigured(
5051            "supervisor was not configured with a forwarding table",
5052        ));
5053    };
5054    let probe_started_at = Instant::now();
5055    let mut deadline = probe_started_at + runtime.health.deadline;
5056    if let Some(drain_deadline) = drain_deadline {
5057        deadline = deadline.min(drain_deadline);
5058    }
5059    let pending = if drain_deadline.is_some() {
5060        forwarding.begin_drain_health_probe_rpc_for(
5061            module_id,
5062            MODULE_CONTROL_OP_HEALTH_CHECK,
5063            probe_started_at,
5064            deadline,
5065        )
5066    } else {
5067        forwarding.begin_health_probe_rpc_for(
5068            module_id,
5069            MODULE_CONTROL_OP_HEALTH_CHECK,
5070            probe_started_at,
5071            deadline,
5072        )
5073    }
5074    .map_err(|err| {
5075        // The endpoint is not registered, so there is no live control lane to
5076        // ask. That is the module being absent, not slow.
5077        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5078    })?;
5079    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5080}
5081
5082/// [`probe_module_health`] for one endpoint rather than the id's active one.
5083///
5084/// A swap probes two processes that no by-id lookup reaches: its candidate
5085/// before cutover, and its superseded incumbent (for busy gauges) while the
5086/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
5087/// bounds the by-id drain probe.
5088async fn probe_endpoint_health(
5089    endpoint: crate::ModuleEndpointId,
5090    runtime: &SupervisorRuntimeConfig,
5091    deadline_cap: Option<Instant>,
5092) -> Result<HealthReport, HealthProbeError> {
5093    let Some(forwarding) = runtime.forwarding.as_ref() else {
5094        return Err(HealthProbeError::misconfigured(
5095            "supervisor was not configured with a forwarding table",
5096        ));
5097    };
5098    let probe_started_at = Instant::now();
5099    let mut deadline = probe_started_at + runtime.health.deadline;
5100    if let Some(cap) = deadline_cap {
5101        deadline = deadline.min(cap);
5102    }
5103    let pending = forwarding
5104        .begin_endpoint_health_probe_rpc_for(
5105            endpoint,
5106            MODULE_CONTROL_OP_HEALTH_CHECK,
5107            probe_started_at,
5108            deadline,
5109        )
5110        .map_err(|err| {
5111            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5112        })?;
5113    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5114}
5115
5116/// Send a begun health probe and classify its answer.
5117async fn await_health_probe(
5118    forwarding: &ForwardingTable,
5119    pending: PendingModuleControlRpc,
5120    deadline: Instant,
5121    probe_budget: Duration,
5122) -> Result<HealthReport, HealthProbeError> {
5123    let PendingModuleControlRpc {
5124        endpoint,
5125        module_sink,
5126        negotiated_ver,
5127        corr,
5128        receiver,
5129    } = pending;
5130    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5131        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5132    })?;
5133    let frame = Frame::build_with_version(
5134        negotiated_ver,
5135        FrameType::Request,
5136        control_flags(),
5137        0,
5138        0,
5139        corr,
5140        body,
5141    )
5142    .map_err(|err| {
5143        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5144    })?;
5145
5146    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
5147    // blocks waiting for capacity when the module's egress queue is full, and an
5148    // unbounded await here freezes the whole supervision actor (it stops polling
5149    // Child::wait and supervisor commands), making the module unrecoverable
5150    // in-band. On timeout the probe fails like any transport failure.
5151    match timeout_at(deadline, module_sink.send(frame)).await {
5152        Ok(Ok(())) => {}
5153        Ok(Err(err)) => {
5154            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5155            // A closed sink means the module's egress channel is gone -- the
5156            // receiving half is dropped when its connection tears down. Proof.
5157            return Err(HealthProbeError::lane_dead(format!(
5158                "failed to send health.check: {err}"
5159            )));
5160        }
5161        Err(_elapsed) => {
5162            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5163            // A full egress queue means the module is not draining its socket, which
5164            // is consistent with a wedged module AND with one whose reader is merely
5165            // starved. Silence, not proof.
5166            return Err(HealthProbeError::no_answer(
5167                "health.check send timed out before enqueue (module egress full)",
5168            ));
5169        }
5170    }
5171
5172    match timeout_at(deadline, receiver).await {
5173        // Each arm records WHAT WAS OBSERVED. Four of them are the module
5174        // demonstrably answering -- rejected, non-health, malformed, wrong op --
5175        // and those prove it is alive even though the probe failed.
5176        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5177            response.health_report().ok_or_else(|| {
5178                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5179            })
5180        }
5181        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5182            format!("health.check rejected: {}", body.message),
5183        )),
5184        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5185            Err(HealthProbeError::lane_dead(message))
5186        }
5187        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5188            Err(HealthProbeError::bad_answer(message))
5189        }
5190        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5191            Err(HealthProbeError::bad_answer(format!(
5192                "expected module-control op '{expected}', got '{actual}'"
5193            )))
5194        }
5195        // A reply that crosses the deadline before this waiter observes it is
5196        // still proof of life. The forwarding path records its end-to-end latency
5197        // before delivering this classification.
5198        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5199            "module answered health.check after its daemon deadline",
5200        )),
5201        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5202            "health.check waiter was canceled before the module responded",
5203        )),
5204        Err(_) => {
5205            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5206            Err(HealthProbeError::no_answer(format!(
5207                "module did not answer health.check within {probe_budget:?}"
5208            )))
5209        }
5210    }
5211}
5212
5213#[allow(clippy::too_many_arguments)]
5214async fn handle_health_report(
5215    spec: &ModuleSpec,
5216    runtime: &SupervisorRuntimeConfig,
5217    registry: &Registry,
5218    process_liveness: &SupervisorProcessLiveness,
5219    snapshot: &SharedSnapshot,
5220    child: &mut Option<SupervisedChild>,
5221    report: HealthReport,
5222    now_ms: u64,
5223) {
5224    let status = supervisor_health_status(report.status);
5225    let detail = report.detail.clone();
5226    let metrics = truncate_health_metrics(report.metrics);
5227    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5228        state.health.status = status;
5229        state.health.last_probe_ms = Some(now_ms);
5230        state.health.detail = detail.clone();
5231        state.health.metrics = metrics.clone();
5232        state.health.consecutive_failures = 0;
5233        state.failed_retry.observe_health(
5234            report.status == HealthStatus::Ok,
5235            runtime.restart_policy.window,
5236        );
5237    });
5238
5239    let action = match report.status {
5240        HealthStatus::Ok => return,
5241        HealthStatus::Degraded => runtime.health.on_degraded,
5242        HealthStatus::Failing => runtime.health.on_failing,
5243    };
5244    apply_l3_health_action(
5245        spec,
5246        runtime,
5247        registry,
5248        process_liveness,
5249        snapshot,
5250        child,
5251        status,
5252        detail.as_deref(),
5253        action,
5254        now_ms,
5255    )
5256    .await;
5257}
5258
5259#[allow(clippy::too_many_arguments)]
5260async fn handle_health_probe_failure(
5261    spec: &ModuleSpec,
5262    runtime: &SupervisorRuntimeConfig,
5263    registry: &Registry,
5264    process_liveness: &SupervisorProcessLiveness,
5265    snapshot: &SharedSnapshot,
5266    child: &mut Option<SupervisedChild>,
5267    err: HealthProbeError,
5268    now_ms: u64,
5269) {
5270    let threshold = runtime.health.failure_threshold.max(1);
5271    let mut failures = 0;
5272    // Carry the evidence class into the operator-visible detail. Without it,
5273    // "module did not answer within 5s" and "the control lane is gone" are two
5274    // prose strings in the same field, and the reader has to know the codebase to
5275    // tell which one is proof of anything.
5276    let detail = format!("[{}] {err}", err.label());
5277    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5278        state
5279            .failed_retry
5280            .observe_health(false, runtime.restart_policy.window);
5281        // A failed wire probe invalidates the last report, even before the
5282        // restart threshold. HTTP probes already mark failures as Failing.
5283        if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5284            state.health.status = SupervisorHealthStatus::Unknown;
5285        }
5286        state.health.last_probe_ms = Some(now_ms);
5287        state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5288        state.health.detail = Some(detail.clone());
5289        state.health.metrics = None;
5290        failures = state.health.consecutive_failures;
5291    });
5292
5293    if failures < threshold {
5294        warn!(
5295            module_id = %spec.module_id,
5296            consecutive_failures = failures,
5297            threshold,
5298            evidence = err.label(),
5299            detail = %detail,
5300            "health.check probe failed"
5301        );
5302        return;
5303    }
5304
5305    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5306        state.state = ModuleState::Unresponsive;
5307        state.health.status = SupervisorHealthStatus::Unresponsive;
5308    });
5309    // The evidence class is logged at the kill site because this is the line an
5310    // operator reads after an unexplained restart. A streak of `no-answer` under
5311    // machine load is the known false-positive shape; a `lane-dead` is not.
5312    if runtime.health.critical {
5313        error!(
5314            module_id = %spec.module_id,
5315            status = "unresponsive",
5316            evidence = err.label(),
5317            detail = %detail,
5318            "critical module health alert"
5319        );
5320    } else {
5321        warn!(
5322            module_id = %spec.module_id,
5323            status = "unresponsive",
5324            evidence = err.label(),
5325            detail = %detail,
5326            "module health threshold breached"
5327        );
5328    }
5329    if let Err(err) = health_restart_child(
5330        spec,
5331        runtime,
5332        registry,
5333        process_liveness,
5334        snapshot,
5335        child,
5336        SupervisorHealthStatus::Unresponsive,
5337        Some(&detail),
5338        now_ms,
5339    )
5340    .await
5341    {
5342        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5343    }
5344}
5345
5346#[allow(clippy::too_many_arguments)]
5347async fn apply_l3_health_action(
5348    spec: &ModuleSpec,
5349    runtime: &SupervisorRuntimeConfig,
5350    registry: &Registry,
5351    process_liveness: &SupervisorProcessLiveness,
5352    snapshot: &SharedSnapshot,
5353    child: &mut Option<SupervisedChild>,
5354    status: SupervisorHealthStatus,
5355    detail: Option<&str>,
5356    action: HealthAction,
5357    now_ms: u64,
5358) {
5359    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5360    match action {
5361        HealthAction::Report => {
5362            info!(
5363                module_id = %spec.module_id,
5364                status = ?status,
5365                detail,
5366                "module reported non-ok health"
5367            );
5368        }
5369        HealthAction::Alert => {
5370            error!(
5371                module_id = %spec.module_id,
5372                status = ?status,
5373                detail,
5374                "module health alert"
5375            );
5376        }
5377        HealthAction::Restart => {
5378            if let Err(err) = health_restart_child(
5379                spec,
5380                runtime,
5381                registry,
5382                process_liveness,
5383                snapshot,
5384                child,
5385                status,
5386                detail,
5387                now_ms,
5388            )
5389            .await
5390            {
5391                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5392            }
5393        }
5394    }
5395}
5396
5397#[allow(clippy::too_many_arguments)]
5398async fn health_restart_child(
5399    spec: &ModuleSpec,
5400    runtime: &SupervisorRuntimeConfig,
5401    registry: &Registry,
5402    process_liveness: &SupervisorProcessLiveness,
5403    snapshot: &SharedSnapshot,
5404    child: &mut Option<SupervisedChild>,
5405    status: SupervisorHealthStatus,
5406    detail: Option<&str>,
5407    now_ms: u64,
5408) -> Result<(), SuperviseError> {
5409    let (enabled, schedule) = {
5410        let mut state = lock_snapshot(snapshot)?;
5411        let enabled = state.enabled;
5412        let schedule = if enabled {
5413            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5414        } else {
5415            None
5416        };
5417        (enabled, schedule)
5418    };
5419
5420    if !enabled {
5421        return Err(SuperviseError::Disabled {
5422            module_id: spec.module_id.clone(),
5423        });
5424    }
5425
5426    if schedule.is_none() {
5427        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5428        error!(
5429            module_id = %spec.module_id,
5430            status = ?status,
5431            detail,
5432            max_restarts = runtime.restart_policy.max_restarts,
5433            window_secs = runtime.restart_policy.window.as_secs(),
5434            reason = %runtime.restart_policy.budget_exhausted_detail(),
5435            "health restart budget exhausted; marking module failed"
5436        );
5437        let stop_notice = begin_forwarding_drain_if_configured(
5438            spec,
5439            runtime,
5440            registry,
5441            snapshot,
5442            Some(true),
5443            RouteCloseReason::Disable,
5444        )
5445        .await?;
5446        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5447            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5448        })?;
5449        let result = drain_optional_child(
5450            &spec.module_id,
5451            spec.protocol,
5452            stop_notice,
5453            registry,
5454            runtime.forwarding.as_deref(),
5455            snapshot,
5456            &runtime.terminal_ring,
5457            &runtime.spawn_events,
5458            child,
5459            runtime.drain_timeout,
5460            ModuleState::Failed,
5461            Some(true),
5462            &runtime.failure_context(true),
5463        )
5464        .await;
5465        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5466            if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5467                state.failed_retry.schedule(&spec.module_id);
5468            }
5469        })?;
5470        result?;
5471        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5472        return Ok(());
5473    }
5474
5475    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5476    let mut restart_count = 0;
5477    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5478        restart_count = state.crash_restarts.len();
5479        state.state = ModuleState::Unresponsive;
5480        state.health.status = status;
5481        state.health.last_action = Some(HealthAction::Restart.to_string());
5482        state.health.last_action_ms = Some(now_ms);
5483    })?;
5484    warn!(
5485        module_id = %spec.module_id,
5486        status = ?status,
5487        detail,
5488        restart_count,
5489        restart_in_window = schedule.restart_in_window,
5490        delay_ms = schedule.delay.as_millis() as u64,
5491        "health-triggered module restart"
5492    );
5493
5494    let stop_notice = begin_forwarding_drain_if_configured(
5495        spec,
5496        runtime,
5497        registry,
5498        snapshot,
5499        Some(true),
5500        RouteCloseReason::Restart,
5501    )
5502    .await?;
5503    drain_optional_child(
5504        &spec.module_id,
5505        spec.protocol,
5506        stop_notice,
5507        registry,
5508        runtime.forwarding.as_deref(),
5509        snapshot,
5510        &runtime.terminal_ring,
5511        &runtime.spawn_events,
5512        child,
5513        runtime.drain_timeout,
5514        ModuleState::Restarting,
5515        Some(true),
5516        &runtime.failure_context(true),
5517    )
5518    .await?;
5519    schedule_respawn(
5520        runtime,
5521        snapshot,
5522        &spec.module_id,
5523        schedule.delay,
5524        RespawnKind::Spawn,
5525        true,
5526    )
5527}
5528
5529fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5530    if let Some(reply) = runtime
5531        .deferred_reload_reply
5532        .lock()
5533        .unwrap_or_else(|p| p.into_inner())
5534        .take()
5535    {
5536        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5537            module_id: module_id.to_string(),
5538            reason: reason.to_string(),
5539        }));
5540    }
5541}
5542
5543fn schedule_respawn(
5544    runtime: &SupervisorRuntimeConfig,
5545    snapshot: &SharedSnapshot,
5546    module_id: &str,
5547    delay: Duration,
5548    kind: RespawnKind,
5549    daemon_initiated: bool,
5550) -> Result<(), SuperviseError> {
5551    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5552    update_snapshot(snapshot, Some(module_id), |state| {
5553        state.respawn_pending = true
5554    })?;
5555    *runtime
5556        .scheduled_respawn
5557        .lock()
5558        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5559        deadline: Instant::now() + delay,
5560        kind,
5561        daemon_initiated,
5562    });
5563    Ok(())
5564}
5565
5566fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5567    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5568        state.health.last_action = Some(action);
5569        state.health.last_action_ms = Some(now_ms);
5570    });
5571}
5572
5573fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5574    match status {
5575        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5576        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5577        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5578    }
5579}
5580
5581/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5582/// returned to every `supervisor.list` and `supervisor.health` caller.
5583///
5584/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5585/// path: that request exists to return a module's complete metrics object, and
5586/// `ck health <module-id>` documents it as the way to see what the cached view
5587/// truncates. The asymmetry is the feature.
5588///
5589/// So a new caller must decide which side it is on rather than assume the cap is
5590/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5591/// the truncation that path exists to avoid.
5592fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5593    let metrics = metrics?;
5594    match serde_json::to_vec(&metrics) {
5595        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5596            "truncated": true,
5597            "original_bytes": encoded.len(),
5598        })),
5599        Ok(_) | Err(_) => Some(metrics),
5600    }
5601}
5602
5603/// Spread health probes so a fleet-wide restart does not converge them.
5604///
5605/// The delay is derived from the module id and probe index rather than a random
5606/// source, so it is deterministic per module: a module keeps its own offset
5607/// across daemon restarts instead of re-rolling into a collision.
5608fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5609    if cadence.is_zero() {
5610        return Duration::ZERO;
5611    }
5612    let cadence_ms = cadence.as_millis() as u64;
5613    // This early return is REDUNDANT, deliberately, and a mutation run will show
5614    // it surviving removal. Recording why here so the next person to notice does
5615    // not have to re-derive it:
5616    //
5617    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5618    //   a zero cadence and builds the Duration from whole milliseconds, so a
5619    //   sub-millisecond cadence cannot come from config.
5620    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5621    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5622    //   -- exactly what this returns.
5623    //
5624    // Kept as a guard against a future widening of the config parser (accepting
5625    // microseconds, say), which would make the sub-millisecond case reachable.
5626    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5627    // divides by zero. Remove this and nothing changes.
5628    if cadence_ms == 0 {
5629        return cadence;
5630    }
5631    // Note that this never returns less than one cadence, including for the FIRST
5632    // probe. So a freshly registered module reports health `unknown` for a full
5633    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5634    // ready to answer.
5635    //
5636    // That is a property of the supervisor's schedule, not of any module: an
5637    // operator watching a restart sees `unknown` and cannot tell it from a module
5638    // that is slow to warm. Measured on two unrelated modules, both flipping to
5639    // `ok` between 22s and 32s after restart.
5640    //
5641    // Left as-is because spreading the first probe is what keeps a fleet-wide
5642    // restart from firing fourteen simultaneous probes into a cold machine. The
5643    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5644    // that thundering herd for a faster first reading.
5645    let jitter_span = (cadence_ms / 10).max(1);
5646    let hash = module_id.as_bytes().iter().fold(
5647        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5648        |acc, byte| {
5649            acc.wrapping_mul(1099511628211)
5650                .wrapping_add(u64::from(*byte))
5651        },
5652    );
5653    cadence + Duration::from_millis(hash % jitter_span)
5654}
5655
5656#[cfg(test)]
5657mod tests {
5658    use super::*;
5659
5660    #[test]
5661    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5662        let handle = SupervisorHandle::new();
5663        let module_id = "readded-tombstone";
5664        handle.record_rescan_removal(module_id);
5665        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5666
5667        handle.apply_identity_configuration(&ModuleSpec {
5668            module_id: module_id.to_string(),
5669            program: PathBuf::from("/test/module"),
5670            args: Vec::new(),
5671            env: Vec::new(),
5672            reserved: false,
5673            reserved_prefixes: Vec::new(),
5674            protocol: ModuleProtocol::Subc,
5675            overlap: Default::default(),
5676        });
5677
5678        assert!(
5679            handle.removal_tombstone_age_ms(module_id).is_none(),
5680            "a re-added module must not retain a stale removal tombstone"
5681        );
5682    }
5683
5684    /// What one module's owner looked like from the control plane at the
5685    /// instant after its first process was spawned.
5686    #[derive(Debug, PartialEq, Eq)]
5687    struct OwnerInSpawnWindow {
5688        module_id: String,
5689        configured: bool,
5690        on_roster: bool,
5691        admission_refusal: Option<&'static str>,
5692    }
5693
5694    /// A supervised module's process can connect, register, sync its scopes
5695    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5696    /// the module on the roster until afterward; configured supervision does
5697    /// so before spawning. In both cases the owner must
5698    /// already read as configured, so a scoped `route.open` against it is
5699    /// refused as retryable `scope_not_synced` and not as terminal
5700    /// `scope_not_live` ("will never sync").
5701    ///
5702    /// The hook runs just after each initial spawn attempt, before returning to
5703    /// the caller, so no race with a real child's registration is needed.
5704    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5705    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5706        use crate::scopes::ScopeTable;
5707        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5708
5709        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5710        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5711            module_id: module_id.to_string(),
5712            program,
5713            args: Vec::new(),
5714            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5715                .into_iter()
5716                .map(|key| (key.to_string(), dir.path().display().to_string()))
5717                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5718                .collect(),
5719            reserved: false,
5720            reserved_prefixes: Vec::new(),
5721            protocol: ModuleProtocol::Subc,
5722            overlap: Default::default(),
5723        };
5724        let live = super::terminal_history_tests::fake_aft_stub_path();
5725        let missing = dir.path().join("definitely-missing-module");
5726
5727        let handle = SupervisorHandle::new();
5728        let mut supervisor =
5729            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5730                .with_handle(handle.clone());
5731        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5732        let hook_handle = handle.clone();
5733        let hook_observed = Arc::clone(&observed);
5734        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5735            // Exactly what the control plane computes for a scoped route.open
5736            // naming this module as the owner of a scope it has not synced.
5737            let configured = hook_handle.is_configured(module_id);
5738            let selector = ScopeSelector::new(
5739                Principal::Reserved {
5740                    module_id: module_id.to_string(),
5741                },
5742                "s".to_string(),
5743                Some(1),
5744            );
5745            let carrier = Principal::Reserved {
5746                module_id: "carrier".to_string(),
5747            };
5748            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5749                .admit(&carrier, module_id, &selector, configured)
5750            {
5751                Ok(_) => None,
5752                Err(refusal) => Some(refusal.code),
5753            };
5754            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5755                module_id: module_id.to_string(),
5756                configured,
5757                on_roster: hook_handle.get(module_id).is_some(),
5758                admission_refusal,
5759            });
5760        })));
5761
5762        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5763        let configured = supervisor
5764            .supervise_configured(stub("configured", live.clone()), true)
5765            .unwrap();
5766        let with_health = supervisor
5767            .supervise_configured_with_health(
5768                stub("with-health", live.clone()),
5769                true,
5770                HealthConfig::default(),
5771                None,
5772                RestartPolicy::default(),
5773            )
5774            .unwrap();
5775        // The failed-spawn path still puts the module on the roster (as
5776        // failed), so it is configured throughout.
5777        let failed = supervisor
5778            .supervise_configured_with_health(
5779                stub("failed-spawn", missing.clone()),
5780                true,
5781                HealthConfig::default(),
5782                None,
5783                RestartPolicy::default(),
5784            )
5785            .unwrap();
5786        // A failed plain `spawn` puts nothing on the roster, so its mark is
5787        // taken back once the spawn has failed.
5788        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5789
5790        let expected = [
5791            "plain",
5792            "configured",
5793            "with-health",
5794            "failed-spawn",
5795            "spawn-error",
5796        ]
5797        .into_iter()
5798        .map(|module_id| OwnerInSpawnWindow {
5799            module_id: module_id.to_string(),
5800            configured: true,
5801            on_roster: !matches!(module_id, "plain" | "spawn-error"),
5802            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5803        })
5804        .collect::<Vec<_>>();
5805        assert_eq!(*observed.lock().unwrap(), expected);
5806
5807        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5808            assert!(
5809                handle.get(module_id).is_some(),
5810                "{module_id} is on the roster"
5811            );
5812            assert!(
5813                handle.is_configured(module_id),
5814                "{module_id} stays configured"
5815            );
5816        }
5817        assert!(handle.get("spawn-error").is_none());
5818        assert!(
5819            !handle.is_configured("spawn-error"),
5820            "a plain spawn that failed must not leave its module marked configured"
5821        );
5822
5823        // Leaving the roster clears the mark with it.
5824        handle.retire("failed-spawn");
5825        assert!(!handle.is_configured("failed-spawn"));
5826
5827        for module in [plain, configured, with_health] {
5828            module.stop().await.unwrap();
5829        }
5830        drop(failed);
5831    }
5832
5833    /// A configured module reads as starting until the supervisor has admitted
5834    /// its first process (exec confirmed, identity durably recorded), even
5835    /// though that process is already running by then and can register over
5836    /// HELLO, which is what lists it in catalog.list. So a client that needs
5837    /// the supervisor state must wait for that state; seeing the module in
5838    /// the catalog does not mean supervisor.list already says running.
5839    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5840    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5841        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5842        let spec = ModuleSpec {
5843            module_id: "spawned-not-admitted".to_string(),
5844            program: super::terminal_history_tests::fake_aft_stub_path(),
5845            args: Vec::new(),
5846            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5847                .into_iter()
5848                .map(|key| (key.to_string(), dir.path().display().to_string()))
5849                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5850                .collect(),
5851            reserved: false,
5852            reserved_prefixes: Vec::new(),
5853            protocol: ModuleProtocol::Subc,
5854            overlap: Default::default(),
5855        };
5856        let handle = SupervisorHandle::new();
5857        let mut supervisor =
5858            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5859                .with_handle(handle.clone());
5860        let observed = Arc::new(Mutex::new(None));
5861        let hook_handle = handle.clone();
5862        let hook_observed = Arc::clone(&observed);
5863        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5864            let state = hook_handle
5865                .get(module_id)
5866                .expect("a configured module is on the roster before it is spawned")
5867                .state()
5868                .unwrap();
5869            *hook_observed.lock().unwrap() = Some(state);
5870        })));
5871
5872        let module = supervisor.supervise_configured(spec, true).unwrap();
5873        assert_eq!(
5874            *observed.lock().unwrap(),
5875            Some(ModuleState::Starting),
5876            "between the spawn and its admission the supervisor still reports starting"
5877        );
5878        assert_eq!(module.state().unwrap(), ModuleState::Running);
5879        module.stop().await.unwrap();
5880    }
5881
5882    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5883        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5884        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5885            snapshot.process_alive = true;
5886            snapshot.pid = Some(41);
5887            snapshot.spawned_at_ms = Some(42);
5888            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5889            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5890            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5891                device: 43,
5892                inode: 44,
5893                #[cfg(windows)]
5894                windows_image: None,
5895            });
5896        })
5897        .unwrap();
5898        snapshot
5899    }
5900
5901    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5902        let snapshot = lock_snapshot(snapshot).unwrap();
5903        assert!(!snapshot.process_alive);
5904        assert_eq!(snapshot.pid, None);
5905        assert_eq!(snapshot.spawned_at_ms, None);
5906        assert_eq!(snapshot.spawned_from, None);
5907        assert_eq!(snapshot.launch_spec_digest, None);
5908        assert_eq!(snapshot.spawned_file_identity, None);
5909    }
5910
5911    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5912    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5913        let supervisor =
5914            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5915        let mut runtime = supervisor.runtime_config();
5916        runtime.test_seed_stale_facts_before_enable_spawn = true;
5917        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5918        let mut child = None;
5919        let spec = ModuleSpec {
5920            module_id: "failed-enable-clears-facts".to_string(),
5921            program: PathBuf::from("/definitely/missing/failed-enable-module"),
5922            args: Vec::new(),
5923            env: Vec::new(),
5924            reserved: false,
5925            reserved_prefixes: Vec::new(),
5926            protocol: ModuleProtocol::Subc,
5927            overlap: Default::default(),
5928        };
5929
5930        let result = set_child_enabled(
5931            &spec,
5932            &runtime,
5933            &supervisor.registry,
5934            &supervisor.process_liveness,
5935            &snapshot,
5936            &mut child,
5937            true,
5938            &runtime.failure_context(false),
5939        )
5940        .await;
5941
5942        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5943        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5944        assert_snapshot_process_facts_cleared(&snapshot);
5945    }
5946
5947    #[tokio::test]
5948    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5949        let supervisor =
5950            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5951        let runtime = supervisor.runtime_config();
5952        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5953            ModuleState::Restarting,
5954            true,
5955        )));
5956        let spec = ModuleSpec {
5957            module_id: "start-stranded-restarting".to_string(),
5958            program: super::terminal_history_tests::fake_aft_stub_path(),
5959            args: Vec::new(),
5960            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5961            reserved: false,
5962            reserved_prefixes: Vec::new(),
5963            protocol: ModuleProtocol::None,
5964            overlap: Default::default(),
5965        };
5966        let mut child = None;
5967        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5968        assert!(!super::set_child_enabled(
5969            &spec,
5970            &runtime,
5971            &Registry::default(),
5972            &supervisor.process_liveness,
5973            &snapshot,
5974            &mut child,
5975            true,
5976            &runtime.failure_context(false),
5977        )
5978        .await
5979        .unwrap());
5980        assert!(child.is_none());
5981        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5982        assert!(super::set_child_enabled(
5983            &spec,
5984            &runtime,
5985            &Registry::default(),
5986            &supervisor.process_liveness,
5987            &snapshot,
5988            &mut child,
5989            true,
5990            &runtime.failure_context(false),
5991        )
5992        .await
5993        .unwrap());
5994        assert_eq!(
5995            lock_snapshot(&snapshot).unwrap().state,
5996            ModuleState::Running
5997        );
5998        let mut child = child.unwrap();
5999        child.start_kill().unwrap();
6000        child.wait().await.unwrap();
6001    }
6002
6003    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6004    async fn failed_reload_spawn_clears_current_process_facts() {
6005        let supervisor =
6006            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6007        let mut runtime = supervisor.runtime_config();
6008        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6009        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6010        let mut child = None;
6011        let spec = ModuleSpec {
6012            module_id: "failed-reload-clears-facts".to_string(),
6013            program: PathBuf::from("/unused/failed-reload-module"),
6014            args: Vec::new(),
6015            env: Vec::new(),
6016            reserved: false,
6017            reserved_prefixes: Vec::new(),
6018            protocol: ModuleProtocol::Subc,
6019            overlap: Default::default(),
6020        };
6021
6022        let result = handle_reload_spawn_failure(
6023            &spec,
6024            &runtime,
6025            &supervisor.process_liveness,
6026            &snapshot,
6027            &mut child,
6028            "forced reload spawn failure".to_string(),
6029        )
6030        .await;
6031
6032        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6033        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6034        assert_snapshot_process_facts_cleared(&snapshot);
6035    }
6036
6037    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6038    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6039        let supervisor =
6040            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6041        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6042        let module = supervisor.supervised_module(
6043            ModuleSpec {
6044                module_id: "drop-clears-facts".to_string(),
6045                program: PathBuf::from("/unused/drop-module"),
6046                args: Vec::new(),
6047                env: Vec::new(),
6048                reserved: false,
6049                reserved_prefixes: Vec::new(),
6050                protocol: ModuleProtocol::Subc,
6051                overlap: Default::default(),
6052            },
6053            supervisor.runtime_config(),
6054            Arc::clone(&snapshot),
6055            None,
6056        );
6057        assert!(!module
6058            .inner
6059            .monitor
6060            .lock()
6061            .unwrap()
6062            .as_ref()
6063            .unwrap()
6064            .is_finished());
6065
6066        drop(module);
6067
6068        assert_eq!(
6069            lock_snapshot(&snapshot).unwrap().state,
6070            ModuleState::Stopped
6071        );
6072        assert_snapshot_process_facts_cleared(&snapshot);
6073    }
6074
6075    #[cfg(unix)]
6076    #[tokio::test]
6077    async fn rescan_preserves_running_protocol_until_respawn() {
6078        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6079        let initial = ModuleSpec {
6080            module_id: "rescan-protocol".into(),
6081            program: PathBuf::from("/bin/sleep"),
6082            args: vec!["60".into()],
6083            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6084                .into_iter()
6085                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6086                .collect(),
6087            reserved: false,
6088            reserved_prefixes: vec![],
6089            protocol: ModuleProtocol::None,
6090            overlap: Default::default(),
6091        };
6092        let supervisor =
6093            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6094        let module = supervisor.spawn(initial.clone()).unwrap();
6095        assert!(module.status().unwrap().live);
6096        let mut next = initial;
6097        next.protocol = ModuleProtocol::Subc;
6098        module
6099            .update_configuration(next.clone(), HealthConfig::default(), None)
6100            .await
6101            .unwrap();
6102        assert!(
6103            module.status().unwrap().live,
6104            "rescan must not require HELLO from the old non-wire process"
6105        );
6106        let runtime = supervisor.runtime_config();
6107        let action = on_child_exit(
6108            &next,
6109            RestartPolicy::default(),
6110            &supervisor.registry,
6111            &module.inner.snapshot,
6112            &runtime.terminal_ring,
6113            &runtime.spawn_events,
6114            &runtime.child_roster,
6115            ExitReport {
6116                kind: ExitKind::Clean,
6117                code: Some(0),
6118                signal: None,
6119                at_ms: unix_ms_now(),
6120            },
6121            true,
6122        )
6123        .await;
6124        assert!(
6125            matches!(action, NextAction::Restart { .. }),
6126            "the old non-wire process's clean exit must restart"
6127        );
6128        module.drain().await.unwrap();
6129    }
6130
6131    #[cfg(unix)]
6132    #[tokio::test]
6133    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6134        use std::os::unix::fs::PermissionsExt;
6135        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6136        let script = dir.join("module.sh");
6137        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6138        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6139        let record_path = dir.join("live-children.json");
6140        let supervisor =
6141            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6142                .with_live_children_record(&record_path);
6143        for (program, args) in [
6144            (PathBuf::from("sleep"), vec!["60".into()]),
6145            (script, vec![]),
6146        ] {
6147            let spec = ModuleSpec {
6148                module_id: "image-identity".into(),
6149                program: program.clone(),
6150                args,
6151                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6152                    .into_iter()
6153                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6154                    .collect(),
6155                reserved: false,
6156                reserved_prefixes: vec![],
6157                protocol: ModuleProtocol::None,
6158                overlap: Default::default(),
6159            };
6160            let module = supervisor.spawn(spec).unwrap();
6161            #[cfg(target_os = "macos")]
6162            {
6163                // SETEXEC confirmation is asynchronous; the orphan record must
6164                // identify the final image, never the intermediate trampoline.
6165                let deadline = Instant::now() + Duration::from_secs(5);
6166                while crate::live_children::read_record(&record_path)
6167                    .unwrap()
6168                    .iter()
6169                    .all(|entry| entry.executable.is_none())
6170                {
6171                    assert!(Instant::now() < deadline, "module image was not confirmed");
6172                    tokio::time::sleep(Duration::from_millis(5)).await;
6173                }
6174            }
6175            let entry = crate::live_children::read_record(&record_path)
6176                .unwrap()
6177                .pop()
6178                .unwrap();
6179            let observed = subc_os::Process::open(entry.pid)
6180                .unwrap()
6181                .unwrap()
6182                .observe()
6183                .unwrap();
6184            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6185            module.drain().await.unwrap();
6186            assert_eq!(
6187                verdict,
6188                crate::live_children::IdentityVerdict::Matches,
6189                "program {program:?}: recorded {entry:?}, observed {observed:?}"
6190            );
6191        }
6192    }
6193
6194    #[cfg(unix)]
6195    fn http_fixture(
6196        dir: &std::path::Path,
6197        url: &str,
6198        threshold: u32,
6199    ) -> crate::daemon_config::ConfiguredModule {
6200        let path = dir.join("subc.jsonc");
6201        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6202            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6203            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6204            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6205            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6206        }}}).to_string()).unwrap();
6207        crate::daemon_config::load(&path)
6208            .unwrap()
6209            .unwrap()
6210            .modules
6211            .pop()
6212            .unwrap()
6213    }
6214
6215    #[cfg(unix)]
6216    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6217        timeout(Duration::from_secs(5), async {
6218            loop {
6219                if module.status().unwrap().health.status == status {
6220                    break;
6221                }
6222                sleep(Duration::from_millis(5)).await;
6223            }
6224        })
6225        .await
6226        .unwrap_or_else(|_| {
6227            panic!(
6228                "expected {status:?}, got {:?}",
6229                module.status().unwrap().health
6230            )
6231        });
6232    }
6233
6234    #[cfg(unix)]
6235    #[tokio::test]
6236    async fn http_health_status_flips_ok_failing_ok() {
6237        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6238        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6239        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6240        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6241        let serving_status = status.clone();
6242        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6243        let server = tokio::spawn(async move {
6244            loop {
6245                let (mut stream, _) = listener.accept().await.unwrap();
6246                let mut request = [0u8; 2048];
6247                let count = stream.read(&mut request).await.unwrap();
6248                assert!(count > 0, "a probe must send an HTTP request");
6249                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6250                let body = if code == 200 {
6251                    "ready"
6252                } else {
6253                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6254                };
6255                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6256                let _ = stream.write_all(response.as_bytes()).await;
6257            }
6258        });
6259        let configured = http_fixture(&dir, &url, 1000);
6260        let module =
6261            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6262                .supervise_configured_with_health(
6263                    configured.module_spec(),
6264                    true,
6265                    configured.health,
6266                    configured.drain_timeout_ms,
6267                    configured.restart,
6268                )
6269                .unwrap();
6270        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6271        status.store(503, std::sync::atomic::Ordering::SeqCst);
6272        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6273        assert!(module
6274            .status()
6275            .unwrap()
6276            .health
6277            .detail
6278            .unwrap()
6279            .contains("scratch failure"));
6280        status.store(200, std::sync::atomic::Ordering::SeqCst);
6281        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6282        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6283        let before = module.status().unwrap();
6284        let (spec, mut health) = module.configuration().unwrap();
6285        health.http = None;
6286        module
6287            .update_configuration(spec.clone(), health.clone(), Some(10))
6288            .await
6289            .unwrap();
6290        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6291        health.http = Some(url);
6292        module
6293            .update_configuration(spec, health, Some(10))
6294            .await
6295            .unwrap();
6296        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6297        assert_eq!(
6298            module.status().unwrap().pid,
6299            before.pid,
6300            "changing a probe must apply live, not restart its process"
6301        );
6302        let (spec, mut health) = module.configuration().unwrap();
6303        health.failure_threshold = 2;
6304        module
6305            .update_configuration(spec, health, Some(10))
6306            .await
6307            .unwrap();
6308        status.store(503, std::sync::atomic::Ordering::SeqCst);
6309        timeout(Duration::from_secs(5), async {
6310            while module.status().unwrap().spawn_generation == before.spawn_generation {
6311                sleep(Duration::from_millis(5)).await;
6312            }
6313        })
6314        .await
6315        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6316        module.drain().await.unwrap();
6317        server.abort();
6318    }
6319
6320    #[cfg(unix)]
6321    #[tokio::test]
6322    async fn http_health_replacement_recovers_after_one_restart() {
6323        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6324        let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6325        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6326        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6327        let mut configured = http_fixture(&dir, &url, 2);
6328        configured.restart.max_restarts = 2;
6329        let module =
6330            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6331                .supervise_configured_with_health(
6332                    configured.module_spec(),
6333                    true,
6334                    configured.health,
6335                    configured.drain_timeout_ms,
6336                    configured.restart,
6337                )
6338                .unwrap();
6339        let first_generation = module.status().unwrap().spawn_generation;
6340        let observed = module.clone();
6341        let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6342        let server = tokio::spawn(async move {
6343            let mut generation = first_generation;
6344            let mut probes = 0;
6345            loop {
6346                let (mut stream, _) = listener.accept().await.unwrap();
6347                let mut request = [0; 2048];
6348                assert!(stream.read(&mut request).await.unwrap() > 0);
6349                let current = observed.status().unwrap().spawn_generation;
6350                if current != generation {
6351                    generation = current;
6352                    probes = 0;
6353                }
6354                probes += 1;
6355                // The original process never recovers. Each replacement has one
6356                // transient miss, then answers successfully for its remaining life.
6357                let healthy = generation != first_generation && probes > 1;
6358                let code = if healthy { 200 } else { 503 };
6359                let response = format!(
6360                    "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6361                );
6362                let _ = stream.write_all(response.as_bytes()).await;
6363                if healthy {
6364                    healthy_tx.send(()).await.unwrap();
6365                }
6366            }
6367        });
6368        timeout(Duration::from_secs(5), async {
6369            let mut healthy_probes = 0;
6370            loop {
6371                if module.state().unwrap() == ModuleState::Failed {
6372                    break;
6373                }
6374                while healthy_rx.try_recv().is_ok() {
6375                    healthy_probes += 1;
6376                }
6377                if healthy_probes >= 3
6378                    && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6379                {
6380                    break;
6381                }
6382                sleep(Duration::from_millis(5)).await;
6383            }
6384        })
6385        .await
6386        .expect("the replacement must settle healthy without exhausting its restart budget");
6387        let status = module.status().unwrap();
6388        module.drain().await.unwrap();
6389        server.abort();
6390        assert_eq!(status.state, ModuleState::Running);
6391        assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6392        assert_eq!(status.health.consecutive_failures, 0);
6393        assert_eq!(status.lifetime_restarts, 1);
6394        assert_eq!(status.restart_count, 1);
6395    }
6396
6397    #[cfg(unix)]
6398    #[tokio::test]
6399    async fn http_health_sustained_failure_exhausts_restart_budget() {
6400        use std::collections::BTreeMap;
6401        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6402        let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6403        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6404        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6405        let mut configured = http_fixture(&dir, &url, 2);
6406        configured.restart.max_restarts = 2;
6407        let module =
6408            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6409                .supervise_configured_with_health(
6410                    configured.module_spec(),
6411                    true,
6412                    configured.health,
6413                    configured.drain_timeout_ms,
6414                    configured.restart,
6415                )
6416                .unwrap();
6417        let observed = module.clone();
6418        let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6419        let served = probes.clone();
6420        let server = tokio::spawn(async move {
6421            loop {
6422                let (mut stream, _) = listener.accept().await.unwrap();
6423                let mut request = [0; 2048];
6424                assert!(stream.read(&mut request).await.unwrap() > 0);
6425                *served
6426                    .lock()
6427                    .unwrap()
6428                    .entry(observed.status().unwrap().spawn_generation)
6429                    .or_default() += 1;
6430                let _ = stream
6431                    .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6432                    .await;
6433            }
6434        });
6435        timeout(Duration::from_secs(5), async {
6436            while module.state().unwrap() != ModuleState::Failed {
6437                sleep(Duration::from_millis(5)).await;
6438            }
6439        })
6440        .await
6441        .expect("sustained failures must still exhaust the health restart budget");
6442        let status = module.status().unwrap();
6443        server.abort();
6444        assert_eq!(status.restart_count, 2);
6445        assert_eq!(status.lifetime_restarts, 2);
6446        assert!(!status.process_alive);
6447        assert_eq!(
6448            probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6449            vec![2, 2, 2]
6450        );
6451    }
6452
6453    #[cfg(unix)]
6454    #[tokio::test]
6455    async fn spawn_clears_the_previous_process_health() {
6456        let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6457        let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6458        configured.health.cadence = Duration::from_secs(3600);
6459        let module =
6460            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6461                .supervise_configured_with_health(
6462                    configured.module_spec(),
6463                    false,
6464                    configured.health,
6465                    configured.drain_timeout_ms,
6466                    configured.restart,
6467                )
6468                .unwrap();
6469        update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6470            state.health.status = SupervisorHealthStatus::Unresponsive;
6471            state.health.consecutive_failures = 3;
6472            state.health.detail = Some("old process refused connections".into());
6473            state.health.metrics = Some(serde_json::json!({"old": true}));
6474            state.health.last_probe_ms = Some(123);
6475            state.health.last_action = Some("restart".into());
6476            state.health.last_action_ms = Some(124);
6477        })
6478        .unwrap();
6479        module.set_enabled(true).await.unwrap();
6480        let health = module.status().unwrap().health;
6481        module.drain().await.unwrap();
6482        assert_eq!(health.consecutive_failures, 0);
6483        assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6484        assert_eq!(health.detail, None);
6485        assert_eq!(health.metrics, None);
6486        assert_eq!(health.last_probe_ms, None);
6487        assert_eq!(health.last_action.as_deref(), Some("restart"));
6488        assert_eq!(health.last_action_ms, Some(124));
6489    }
6490
6491    #[cfg(unix)]
6492    #[tokio::test]
6493    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6494        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6495        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6496        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6497        drop(unused);
6498        let configured = http_fixture(&dir, &url, 2);
6499        let module =
6500            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6501                .supervise_configured_with_health(
6502                    configured.module_spec(),
6503                    true,
6504                    configured.health,
6505                    configured.drain_timeout_ms,
6506                    configured.restart,
6507                )
6508                .unwrap();
6509        let before = module.status().unwrap().spawn_generation;
6510        timeout(Duration::from_secs(5), async {
6511            loop {
6512                let status = module.status().unwrap();
6513                if status.spawn_generation > before {
6514                    assert!(status.lifetime_restarts > 0);
6515                    break;
6516                }
6517                sleep(Duration::from_millis(5)).await;
6518            }
6519        })
6520        .await
6521        .expect("sustained HTTP refusal must trigger the health restart policy");
6522        module.drain().await.unwrap();
6523    }
6524
6525    #[cfg(unix)]
6526    #[tokio::test]
6527    async fn http_health_timeout_honours_deadline() {
6528        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6529        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6530        let server = tokio::spawn(async move {
6531            let _held = listener.accept().await.unwrap();
6532            std::future::pending::<()>().await;
6533        });
6534        let error = timeout(
6535            Duration::from_secs(1),
6536            probe_http_health(&url, Duration::from_millis(10)),
6537        )
6538        .await
6539        .expect("the probe must enforce its own deadline")
6540        .unwrap_err();
6541        server.abort();
6542        assert!(error.to_string().contains("timed out"));
6543    }
6544
6545    #[cfg(unix)]
6546    #[tokio::test]
6547    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6548        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6549        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6550        let url = format!(
6551            "http://localhost:{}/healthz",
6552            listener.local_addr().unwrap().port()
6553        );
6554        let server = tokio::spawn(async move {
6555            let (mut stream, _) = listener.accept().await.unwrap();
6556            let mut request = [0u8; 2048];
6557            assert!(stream.read(&mut request).await.unwrap() > 0);
6558            stream
6559                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6560                .await
6561                .unwrap();
6562        });
6563        // The deadline only bounds a hang. A probe that never tried the IPv6
6564        // address would be refused on 127.0.0.1 and fail at once, so a longer
6565        // deadline does not weaken the assertion; one second timed out under a
6566        // loaded parallel test run.
6567        assert_eq!(
6568            probe_http_health(&url, Duration::from_secs(10))
6569                .await
6570                .unwrap()
6571                .status,
6572            HealthStatus::Ok
6573        );
6574        server.await.unwrap();
6575    }
6576
6577    #[cfg(unix)]
6578    #[tokio::test]
6579    async fn http_health_timeout_keeps_partial_status_and_body() {
6580        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6581        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6582        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6583        let server = tokio::spawn(async move {
6584            let (mut stream, _) = listener.accept().await.unwrap();
6585            let mut request = [0u8; 2048];
6586            assert!(stream.read(&mut request).await.unwrap() > 0);
6587            stream
6588                .write_all(
6589                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6590                )
6591                .await
6592                .unwrap();
6593            std::future::pending::<()>().await;
6594        });
6595        let error = probe_http_health(&url, Duration::from_secs(1))
6596            .await
6597            .unwrap_err()
6598            .to_string();
6599        server.abort();
6600        assert!(
6601            error.contains("timed out")
6602                && error.contains("503 Unavailable")
6603                && error.contains("partial diagnostic"),
6604            "{error}"
6605        );
6606    }
6607
6608    #[cfg(unix)]
6609    #[tokio::test]
6610    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6611        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6612        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6613        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6614        let server = tokio::spawn(async move {
6615            let (mut stream, _) = listener.accept().await.unwrap();
6616            let mut request = [0u8; 2048];
6617            assert!(stream.read(&mut request).await.unwrap() > 0);
6618            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6619            let response = format!(
6620                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6621                body.len()
6622            );
6623            stream.write_all(response.as_bytes()).await.unwrap();
6624        });
6625        let error = probe_http_health(&url, Duration::from_secs(1))
6626            .await
6627            .unwrap_err()
6628            .to_string();
6629        server.await.unwrap();
6630        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6631        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6632        assert!(!error.contains("not-in-diagnostic"));
6633    }
6634
6635    #[cfg(unix)]
6636    #[tokio::test]
6637    async fn http_health_real_nats_server_monitoring() {
6638        if std::process::Command::new("nats-server")
6639            .arg("--version")
6640            .env("XDG_DATA_HOME", std::env::temp_dir())
6641            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6642            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6643            .output()
6644            .is_err()
6645        {
6646            eprintln!(
6647                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6648            );
6649            return;
6650        }
6651        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6652        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6653        let port = monitor.local_addr().unwrap().port();
6654        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6655        let client_port = client.local_addr().unwrap().port();
6656        let config = dir.join("server.conf");
6657        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6658        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6659        configured.program = PathBuf::from("nats-server");
6660        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6661        drop(monitor);
6662        drop(client);
6663        let module =
6664            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6665                .supervise_configured_with_health(
6666                    configured.module_spec(),
6667                    true,
6668                    configured.health,
6669                    configured.drain_timeout_ms,
6670                    configured.restart,
6671                )
6672                .unwrap();
6673        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6674        module.drain().await.unwrap();
6675    }
6676
6677    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6678    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6679        let supervisor =
6680            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6681        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6682        let initial = ModuleSpec {
6683            module_id: "rescan-preserves-spawn-facts".to_string(),
6684            program: PathBuf::from("/spawned/module"),
6685            args: Vec::new(),
6686            env: Vec::new(),
6687            reserved: false,
6688            reserved_prefixes: Vec::new(),
6689            protocol: ModuleProtocol::Subc,
6690            overlap: Default::default(),
6691        };
6692        let module = supervisor.supervised_module(
6693            initial.clone(),
6694            supervisor.runtime_config(),
6695            snapshot,
6696            None,
6697        );
6698        let before = module.status().unwrap();
6699        let mut replacement = initial;
6700        replacement.program = PathBuf::from("/rescanned/replacement-module");
6701
6702        module
6703            .update_configuration(replacement, HealthConfig::default(), None)
6704            .await
6705            .unwrap();
6706
6707        let after = module.status().unwrap();
6708        assert_eq!(after.pid, before.pid);
6709        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6710        assert_eq!(after.spawned_from, before.spawned_from);
6711        drop(module);
6712    }
6713}
6714
6715fn unix_ms_now() -> u64 {
6716    SystemTime::now()
6717        .duration_since(UNIX_EPOCH)
6718        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6719        .unwrap_or(0)
6720}
6721
6722async fn supervise_loop(
6723    mut spec: ModuleSpec,
6724    mut runtime: SupervisorRuntimeConfig,
6725    registry: Arc<Registry>,
6726    process_liveness: Arc<SupervisorProcessLiveness>,
6727    snapshot: SharedSnapshot,
6728    mut child: Option<SupervisedChild>,
6729    mut commands: mpsc::Receiver<SupervisorCommand>,
6730) {
6731    let mut health_probe = HealthProbeRuntime::default();
6732    // Registry writes (including embedded callers) notify this module only.
6733    // Subscribe before the first refresh; watch retains changes that arrive
6734    // while commands or probes are running. No polling fallback is needed.
6735    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6736        Ok(changes) => Some(changes),
6737        Err(err) => {
6738            // A poisoned registry cannot accept further writes, so it cannot
6739            // recover via a timer. Keep exit and command handling alive.
6740            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6741            None
6742        }
6743    };
6744    // All restart backoffs run here, including health and operator requests.
6745    // While one is pending the loop serves commands, so disable or drain can
6746    // cancel the replacement without spawning a process just to stop it.
6747    let mut pending_respawn: Option<PendingRespawn> = None;
6748    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
6749    // before anything else so a stop that interrupted a swap runs at once.
6750    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6751    let mut previous_instance_recheck = tokio::time::interval_at(
6752        Instant::now() + Duration::from_secs(5),
6753        Duration::from_secs(5),
6754    );
6755    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6756    loop {
6757        #[cfg(test)]
6758        {
6759            lock_snapshot(&snapshot).unwrap().actor_select = None;
6760        }
6761        #[cfg(target_os = "macos")]
6762        if let Some(active) = child.as_mut() {
6763            active.confirm_privacy_exec().await;
6764        }
6765        if let Some(scheduled) = runtime
6766            .scheduled_respawn
6767            .lock()
6768            .unwrap_or_else(|p| p.into_inner())
6769            .take()
6770        {
6771            runtime.recovery_is_daemon = scheduled.daemon_initiated;
6772            pending_respawn = Some(scheduled);
6773        }
6774        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6775            pending_respawn = None;
6776            cancel_deferred_reload(
6777                &runtime,
6778                &spec.module_id,
6779                "respawn cancelled by a supervisor command",
6780            );
6781        }
6782        if child.is_none()
6783            && pending_respawn.is_none()
6784            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6785        {
6786            cancel_deferred_reload(
6787                &runtime,
6788                &spec.module_id,
6789                "respawn cancelled before a replacement was spawned",
6790            );
6791            let mut stranded = false;
6792            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6793                state.respawn_pending = false;
6794                state.coalesced_restart_pending = false;
6795                if matches!(
6796                    state.state,
6797                    ModuleState::Restarting
6798                        | ModuleState::Starting
6799                        | ModuleState::Draining
6800                        | ModuleState::Unresponsive
6801                ) {
6802                    error!(module_id = %spec.module_id, state = ?state.state,
6803                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6804                    stranded = true;
6805                }
6806            });
6807            if stranded {
6808                fail_snapshot(
6809                    &snapshot,
6810                    Some(&spec.module_id),
6811                    None,
6812                    &runtime.failure_context(runtime.recovery_is_daemon),
6813                );
6814            }
6815        }
6816        if let Some(command) = requeued.pop_front() {
6817            if !handle_supervisor_command(
6818                command,
6819                &mut spec,
6820                &mut runtime,
6821                &registry,
6822                &process_liveness,
6823                &snapshot,
6824                &mut child,
6825                &mut commands,
6826                &mut requeued,
6827            )
6828            .await
6829            {
6830                return;
6831            }
6832            if child.is_some() || !respawn_still_pending(&snapshot) {
6833                pending_respawn = None;
6834                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6835                    state.respawn_pending = false
6836                });
6837            }
6838            continue;
6839        }
6840        if child.is_some() {
6841            runtime.recovery_is_daemon = true;
6842            #[cfg(test)]
6843            {
6844                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6845            }
6846            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
6847            let no_checks = runtime.health.http.is_none()
6848                && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
6849                    || (health_probe.registered_connection.is_some() && !health_probe.advertised));
6850            let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
6851                state
6852                    .failed_retry
6853                    .uptime_reset_at(no_checks, runtime.restart_policy.window)
6854            });
6855            let wake_at = match (health_probe.wake_at(), uptime_reset) {
6856                (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
6857                (probe, None) => probe,
6858            };
6859            #[cfg(test)]
6860            let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
6861            let probe_sleep = supervisor_sleep(wake_at);
6862            tokio::pin!(probe_sleep);
6863            // A `protocol: "none"` child (a plain process that never registers
6864            // over the subc wire, such as nats-server) with no HTTP health
6865            // check has nothing to probe, so it waits only for its exit or a
6866            // supervisor command. A rescan that adds an HTTP check is a command.
6867            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6868            #[cfg(test)]
6869            {
6870                let registration_pending = wire_child
6871                    && registration_changes
6872                        .as_ref()
6873                        .is_some_and(|changes| changes.has_changed().unwrap());
6874                let mut state = lock_snapshot(&snapshot).unwrap();
6875                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6876                    generation: state.spawn_generation,
6877                    turn: state.actor_turns,
6878                    registered_connection: health_probe.registered_connection,
6879                    next_probe_at: health_probe.next_probe_at,
6880                    wake_after,
6881                });
6882            }
6883            let active_child = child.as_mut().expect("child checked above");
6884            tokio::select! {
6885                wait_result = active_child.wait() => {
6886                    if let Ok(mut state) = lock_snapshot(&snapshot) {
6887                        state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
6888                    }
6889                    #[cfg(test)]
6890                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6891                    // Every arm below that gives up on the CHILD must keep the
6892                    // supervision task itself alive (child = None, loop
6893                    // continues into command-serving mode). Returning here
6894                    // closes the command channel, which makes the module
6895                    // permanently unrestartable in-band: a clean child exit
6896                    // of an enabled module once wedged the fleet this way
6897                    // ('supervisor command channel is closed') and required a
6898                    // full daemon restart to recover.
6899                    let exit_report = match wait_result {
6900                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6901                        Err(err) => {
6902                            active_child.drain_stderr(&spec.module_id).await;
6903                            fail_after_wait_error(
6904                                &spec.module_id,
6905                                &snapshot,
6906                                &runtime.terminal_ring,
6907                                &runtime.spawn_events,
6908                                format!("failed to wait for supervised module: {err}"),
6909                            &runtime.failure_context(true),
6910                            );
6911                            untrack_if_registration_released(
6912                                &process_liveness,
6913                                &registry,
6914                                &spec.module_id,
6915                                &snapshot,
6916                            );
6917                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6918                            child = None;
6919                            continue;
6920                        }
6921                    };
6922                    active_child.drain_stderr(&spec.module_id).await;
6923
6924                    let next = on_child_exit(
6925                        &spec,
6926                        runtime.restart_policy,
6927                        &registry,
6928                        &snapshot,
6929                        &runtime.terminal_ring,
6930                        &runtime.spawn_events,
6931                        &runtime.child_roster,
6932                        exit_report,
6933                        true,
6934                    ).await;
6935                    runtime.start_requester = None;
6936                    runtime.pending_activity_restart = false;
6937                    // The exit is recorded, so a daemon shutdown may stop
6938                    // waiting for this child (see `SupervisedChild::wait`).
6939                    active_child.release_roster();
6940                    match next {
6941                        NextAction::Stop { registration_released } => {
6942                            if registration_released {
6943                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6944                            }
6945                            child = None;
6946                        }
6947                        NextAction::Restart { schedule } => {
6948                            let delay = schedule.map_or(
6949                                runtime.restart_policy.delay_for_restart(0),
6950                                |schedule| schedule.delay,
6951                            );
6952                            if let Some(schedule) = schedule {
6953                                log_crash_respawn(&spec.module_id, schedule);
6954                            }
6955                            // The exited child is fully recorded at this point,
6956                            // so release it and count the backoff down in the
6957                            // command-serving branch below rather than sleeping
6958                            // here: commands cannot be received from inside this
6959                            // select arm, and an operator disable or drain that
6960                            // arrives during the backoff must cancel the pending
6961                            // respawn instead of waiting for it to spawn first.
6962                            child = None;
6963                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
6964                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6965                        }
6966                    }
6967                }
6968                command = commands.recv() => {
6969                    #[cfg(test)]
6970                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6971                    let Some(command) = command else {
6972                        return;
6973                    };
6974                    if !handle_supervisor_command(
6975                        command,
6976                        &mut spec,
6977                        &mut runtime,
6978                        &registry,
6979                        &process_liveness,
6980                        &snapshot,
6981                        &mut child,
6982                        &mut commands,
6983                        &mut requeued,
6984                    ).await {
6985                        return;
6986                    }
6987                }
6988                _ = async {
6989                    match registration_changes.as_mut() {
6990                        Some(changes) => { let _ = changes.changed().await; }
6991                        None => std::future::pending().await,
6992                    }
6993                }, if wire_child => {
6994                    #[cfg(test)]
6995                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6996                }
6997                _ = &mut probe_sleep => {
6998                    #[cfg(test)]
6999                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7000                    if health_probe.due() {
7001                        run_health_probe_cycle(
7002                            &spec,
7003                            &runtime,
7004                            &registry,
7005                            &process_liveness,
7006                            &snapshot,
7007                            &mut child,
7008                        ).await;
7009                        if child.is_none() {
7010                            runtime.start_requester = None;
7011                            runtime.pending_activity_restart = false;
7012                        }
7013                        if child.is_some() {
7014                            health_probe.schedule_next(&spec, runtime.health.cadence);
7015                        }
7016                    }
7017                }
7018            }
7019        } else if let Some(pending) = pending_respawn {
7020            tokio::select! {
7021                _ = sleep_until(pending.deadline) => {
7022                    pending_respawn = None;
7023                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7024                    // A command handled below while the backoff elapsed may
7025                    // have stopped the module; never respawn past an operator's
7026                    // disable or drain.
7027                    if !respawn_still_pending(&snapshot) {
7028                        continue;
7029                    }
7030                    // The daemon began shutting down during the backoff: the
7031                    // spawn would be refused anyway, and refusing it here
7032                    // leaves the module stopped instead of reporting a
7033                    // failed restart.
7034                    if runtime.child_roster.is_closed() {
7035                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7036                            state.state = ModuleState::Stopped;
7037                        });
7038                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7039                        continue;
7040                    }
7041                    if let Err(err) = release_dead_registration(
7042                        &registry,
7043                        runtime.forwarding.as_deref(),
7044                        &snapshot,
7045                        &spec.module_id,
7046                        &runtime.failure_context(runtime.recovery_is_daemon),
7047                    ).await {
7048                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7049                        continue;
7050                    }
7051
7052                    if matches!(pending.kind, RespawnKind::Reload) {
7053                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7054                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
7055                        if result.is_ok() && runtime.pending_activity_restart {
7056                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7057                        }
7058                        runtime.pending_activity_restart = false;
7059                        if let Some(reply) = reply { let _ = reply.send(result); }
7060                        continue;
7061                    }
7062                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7063                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7064                        Ok(next_child) => {
7065                            child = Some(next_child);
7066                            if runtime.pending_activity_restart {
7067                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7068                                runtime.pending_activity_restart = false;
7069                            }
7070                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7071                        }
7072                        Err(err) => {
7073                            fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7074                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7075                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7076                        }
7077                    }
7078                }
7079                command = commands.recv() => {
7080                    let Some(command) = command else {
7081                        return;
7082                    };
7083                    if !handle_supervisor_command(
7084                        command,
7085                        &mut spec,
7086                        &mut runtime,
7087                        &registry,
7088                        &process_liveness,
7089                        &snapshot,
7090                        &mut child,
7091                        &mut commands,
7092                        &mut requeued,
7093                    ).await {
7094                        return;
7095                    }
7096                    // Reconcile the pending respawn with what the command did:
7097                    // a start may already have spawned a fresh child,
7098                    // while a disable or drain moved the snapshot out of the
7099                    // state the respawn was counting down from.
7100                    if child.is_some() || !respawn_still_pending(&snapshot) {
7101                        pending_respawn = None;
7102                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7103                    }
7104                }
7105            }
7106        } else {
7107            let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7108                (state.enabled && state.state == ModuleState::Failed)
7109                    .then_some(state.failed_retry.pending)
7110                    .flatten()
7111            });
7112            let command = if lock_snapshot(&snapshot)
7113                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7114            {
7115                tokio::select! {
7116                    command = commands.recv() => command,
7117                    _ = previous_instance_recheck.tick() => {
7118                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7119                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7120                            if enabled && !runtime.child_roster.is_closed() {
7121                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7122                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7123                                    Ok(next_child) => child = Some(next_child),
7124                                    Err(error) => {
7125                                        fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7126                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7127                                    }
7128                                }
7129                            } else {
7130                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7131                                    state.waiting_for_previous_pid = None;
7132                                    state.waiting_for_previous_detail = None;
7133                                });
7134                            }
7135                        }
7136                        continue;
7137                    }
7138                }
7139            } else {
7140                tokio::select! {
7141                    biased;
7142                    command = commands.recv() => command,
7143                    _ = async {
7144                        match slow_retry {
7145                            Some(retry) => sleep_until(retry.deadline).await,
7146                            None => std::future::pending().await,
7147                        }
7148                    } => {
7149                        let retry = slow_retry.expect("only a scheduled slow retry can wake");
7150                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7151                            state.failed_retry.pending = None;
7152                        });
7153                        if runtime.child_roster.is_closed() {
7154                            continue;
7155                        }
7156                        info!(module_id = %spec.module_id, attempt = retry.attempt,
7157                            delay_secs = retry.delay.as_secs(), "starting failed module retry");
7158                        if let Err(error) = set_child_enabled(
7159                            &spec, &runtime, &registry, &process_liveness, &snapshot, &mut child, true,
7160                            &runtime.failure_context(true),
7161                        ).await {
7162                            error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7163                        }
7164                        continue;
7165                    }
7166                }
7167            };
7168            let Some(command) = command else {
7169                return;
7170            };
7171            if !handle_supervisor_command(
7172                command,
7173                &mut spec,
7174                &mut runtime,
7175                &registry,
7176                &process_liveness,
7177                &snapshot,
7178                &mut child,
7179                &mut commands,
7180                &mut requeued,
7181            )
7182            .await
7183            {
7184                return;
7185            }
7186        }
7187    }
7188}
7189
7190// Preserve the deadline until the future is polled. Starting a relative sleep
7191// after the executor yields would restart part of the countdown.
7192async fn supervisor_sleep(wake_at: Option<Instant>) {
7193    match wake_at {
7194        Some(deadline) => sleep_until(deadline).await,
7195        None => std::future::pending().await,
7196    }
7197}
7198
7199fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7200    info!(
7201        module_id,
7202        restart_in_window = schedule.restart_in_window,
7203        delay_ms = schedule.delay.as_millis() as u64,
7204        "respawning after crash"
7205    );
7206}
7207
7208/// Whether the respawn a backoff was counting down to is still wanted. A
7209/// disable or drain handled while the backoff elapsed moves the snapshot out
7210/// of `Restarting`, and the operator's stop must win over the pending respawn,
7211/// so every sleep-then-spawn path re-validates against the live snapshot
7212/// instead of assuming the state it left behind still holds.
7213fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7214    matches!(
7215        lock_snapshot(snapshot),
7216        Ok(state) if state.enabled && state.state == ModuleState::Restarting
7217    )
7218}
7219
7220enum NextAction {
7221    Stop {
7222        registration_released: bool,
7223    },
7224    Restart {
7225        schedule: Option<CrashRestartSchedule>,
7226    },
7227}
7228
7229#[allow(clippy::too_many_arguments)]
7230async fn handle_supervisor_command(
7231    command: SupervisorCommand,
7232    spec: &mut ModuleSpec,
7233    runtime: &mut SupervisorRuntimeConfig,
7234    registry: &Arc<Registry>,
7235    process_liveness: &SupervisorProcessLiveness,
7236    snapshot: &SharedSnapshot,
7237    child: &mut Option<SupervisedChild>,
7238    commands: &mut mpsc::Receiver<SupervisorCommand>,
7239    requeued: &mut VecDeque<SupervisorCommand>,
7240) -> bool {
7241    if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7242        runtime.recovery_is_daemon = false;
7243    }
7244    if matches!(
7245        command,
7246        SupervisorCommand::Drain { .. }
7247            | SupervisorCommand::Retire { .. }
7248            | SupervisorCommand::SetEnabled { .. }
7249            | SupervisorCommand::Restart { .. }
7250    ) {
7251        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7252            state.failed_retry.cancel()
7253        });
7254    }
7255    let command =
7256        if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
7257            match command {
7258                SupervisorCommand::Restart { reply, .. }
7259                | SupervisorCommand::Reload { reply, .. }
7260                | SupervisorCommand::Swap { reply, .. } => {
7261                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7262                    return true;
7263                }
7264                SupervisorCommand::SetEnabled {
7265                    enabled: true,
7266                    reply,
7267                    ..
7268                } => {
7269                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7270                    return true;
7271                }
7272                other => other,
7273            }
7274        } else {
7275            command
7276        };
7277    match command {
7278        SupervisorCommand::Drain { requester, reply } => {
7279            // A plain stop runs no forwarding drain, so nothing reaches the
7280            // module over its connection before the wait: ask by signal.
7281            let result = drain_optional_child(
7282                &spec.module_id,
7283                spec.protocol,
7284                StopNotice::NotSent,
7285                registry,
7286                runtime.forwarding.as_deref(),
7287                snapshot,
7288                &runtime.terminal_ring,
7289                &runtime.spawn_events,
7290                child,
7291                runtime.drain_timeout,
7292                ModuleState::Stopped,
7293                None,
7294                &FailureContext::operator(),
7295            )
7296            .await;
7297            let registration_released = result.is_ok();
7298            if registration_released {
7299                runtime.record_activity_transition(
7300                    "subc.module_stopped",
7301                    &spec.module_id,
7302                    requester,
7303                );
7304            }
7305            let _ = reply.send(result);
7306            if registration_released {
7307                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7308            }
7309            false
7310        }
7311        SupervisorCommand::Retire { requester, reply } => {
7312            let result = async {
7313                let stop_notice = begin_forwarding_drain_if_configured(
7314                    spec,
7315                    runtime,
7316                    registry,
7317                    snapshot,
7318                    None,
7319                    RouteCloseReason::Disable,
7320                )
7321                .await?;
7322                drain_optional_child(
7323                    &spec.module_id,
7324                    spec.protocol,
7325                    stop_notice,
7326                    registry,
7327                    runtime.forwarding.as_deref(),
7328                    snapshot,
7329                    &runtime.terminal_ring,
7330                    &runtime.spawn_events,
7331                    child,
7332                    runtime.drain_timeout,
7333                    ModuleState::Stopped,
7334                    None,
7335                    &FailureContext::operator(),
7336                )
7337                .await
7338            }
7339            .await;
7340            let registration_released = result.is_ok();
7341            if registration_released {
7342                runtime.record_activity_transition(
7343                    "subc.module_stopped",
7344                    &spec.module_id,
7345                    requester,
7346                );
7347            }
7348            let _ = reply.send(result);
7349            if registration_released {
7350                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7351            }
7352            false
7353        }
7354        SupervisorCommand::Restart {
7355            requester,
7356            drain_timeout_ms,
7357            received_at_generation,
7358            queued_at,
7359            reply,
7360        } => {
7361            // Without this line a restart that waited in the queue (behind a
7362            // health probe cycle or another command) was invisible: the log
7363            // showed only the drain timing out, minutes after the operator's call.
7364            info!(
7365                module_id = %spec.module_id,
7366                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7367                "restart command dequeued"
7368            );
7369            // ACK AT INITIATION, not completion. The blocking form deadlocked any
7370            // caller whose own request lane rides the module being restarted: the
7371            // caller's in-flight request keeps the drain from quiescing, the drain
7372            // keeps the restart from completing, and the completion keeps the reply
7373            // from releasing the caller — so the drain always timed out and cut the
7374            // initiator with a GOODBYE, even on a healthy module. Replying once the
7375            // restart is validated lets a self-lane caller settle, which is exactly
7376            // what makes the drain succeed. Completion is observable via
7377            // supervisor.list / module status; a post-ack failure lands the module
7378            // in a visible terminal state below rather than in a reply nobody can
7379            // receive.
7380            let validation = match lock_snapshot(snapshot) {
7381                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7382                    module_id: spec.module_id.clone(),
7383                }),
7384                Ok(_) => Ok(()),
7385                Err(err) => Err(err),
7386            };
7387            let initiated = validation.is_ok();
7388            let _ = reply.send(validation);
7389            // A restart asks for a fresh process. Commands run one at a time,
7390            // so a restart queued behind another restart (two operator calls
7391            // in quick succession) is dequeued the moment the first one has
7392            // spawned its replacement -- before that process has sent HELLO.
7393            // Running it would drain and kill the process the first restart
7394            // just produced, which is the opposite of what both callers asked
7395            // for. If a process spawned after this request was received is
7396            // still supervised, the request is already satisfied. Not when the
7397            // configuration changed since that spawn: then the newer process
7398            // predates the spec this restart may exist to apply.
7399            // Generation publication can lag a child that is already serving
7400            // while its identity is recorded. Only process creation after request
7401            // receipt satisfies a restart, not publication after receipt.
7402            let satisfied_by_generation = if initiated
7403                && child
7404                    .as_ref()
7405                    .is_some_and(|child| child.spawned_at > queued_at)
7406            {
7407                lock_snapshot(snapshot).ok().and_then(|state| {
7408                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7409                })
7410            } else {
7411                None
7412            };
7413            let satisfied_by_pending = initiated
7414                && child.is_none()
7415                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7416                    // No child exists yet, so this replacement will necessarily
7417                    // be created after the queued request, unlike a published
7418                    // generation whose process may already have been serving.
7419                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7420                    if pending {
7421                        state.coalesced_restart_pending = true;
7422                    }
7423                    pending
7424                });
7425            if satisfied_by_pending {
7426                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7427            } else if let Some(generation) = satisfied_by_generation {
7428                info!(
7429                    module_id = %spec.module_id,
7430                    received_at_generation,
7431                    "restart already satisfied by generation {generation}; not restarting again"
7432                );
7433            } else if initiated {
7434                // Precedence: this restart's operator override, else the module's
7435                // configured budget (already resolved into the runtime).
7436                let drain_timeout = drain_timeout_ms
7437                    .map(Duration::from_millis)
7438                    .unwrap_or(runtime.drain_timeout);
7439                if let Err(err) = restart_child(
7440                    spec,
7441                    runtime,
7442                    registry,
7443                    process_liveness,
7444                    snapshot,
7445                    child,
7446                    drain_timeout,
7447                )
7448                .await
7449                {
7450                    warn!(
7451                        module_id = %spec.module_id,
7452                        error = %err,
7453                        "operator restart failed after initiation ack; module state carries the outcome"
7454                    );
7455                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7456                        state.state = ModuleState::Failed;
7457                        clear_current_process_facts(state);
7458                    });
7459                } else {
7460                    runtime.start_requester = requester;
7461                    runtime.pending_activity_restart = true;
7462                }
7463            }
7464            true
7465        }
7466        SupervisorCommand::Reload { requester, reply } => {
7467            let result =
7468                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7469            if result.is_ok() {
7470                runtime.start_requester = requester;
7471                runtime.pending_activity_restart = true;
7472            }
7473            if result.is_ok()
7474                && runtime
7475                    .scheduled_respawn
7476                    .lock()
7477                    .unwrap_or_else(|p| p.into_inner())
7478                    .as_ref()
7479                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7480            {
7481                *runtime
7482                    .deferred_reload_reply
7483                    .lock()
7484                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7485            } else {
7486                let _ = reply.send(result);
7487            }
7488            true
7489        }
7490        SupervisorCommand::SetEnabled {
7491            enabled,
7492            requester,
7493            reply,
7494        } => {
7495            let previous_requester = runtime.start_requester.clone();
7496            if enabled {
7497                runtime.start_requester = requester.clone();
7498            }
7499            let result = set_child_enabled(
7500                spec,
7501                runtime,
7502                registry,
7503                process_liveness,
7504                snapshot,
7505                child,
7506                enabled,
7507                &runtime.failure_context(false),
7508            )
7509            .await;
7510            if matches!(result, Ok(true)) {
7511                runtime.pending_activity_restart = false;
7512                if enabled {
7513                    runtime.start_requester = requester;
7514                } else {
7515                    runtime.start_requester = None;
7516                    runtime.record_activity_transition(
7517                        "subc.module_disabled",
7518                        &spec.module_id,
7519                        requester,
7520                    );
7521                }
7522            } else {
7523                runtime.start_requester = previous_requester;
7524            }
7525            let _ = reply.send(result);
7526            true
7527        }
7528        SupervisorCommand::UpdateConfiguration {
7529            spec: next_spec,
7530            health,
7531            drain_timeout_ms,
7532            reply,
7533        } => {
7534            if let Some(handle) = &runtime.supervisor_handle {
7535                handle.apply_identity_configuration(&next_spec);
7536            }
7537            *spec = next_spec;
7538            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7539                state.configuration_updated_since_spawn = true;
7540            });
7541            let health_changed = runtime.health != health;
7542            runtime.health = health;
7543            // Reset the cadence and old endpoint's failure streak on a live
7544            // health-policy change rather than waiting for its old deadline.
7545            if health_changed {
7546                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7547                    state.health = ModuleHealthStatus::default();
7548                });
7549            }
7550            runtime.drain_timeout = drain_timeout_ms
7551                .map(Duration::from_millis)
7552                .unwrap_or(runtime.default_drain_timeout);
7553            *runtime
7554                .effective_drain_timeout
7555                .lock()
7556                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7557            let _ = reply.send(());
7558            true
7559        }
7560        SupervisorCommand::Swap {
7561            requester,
7562            ready_timeout,
7563            reply,
7564        } => {
7565            let mut swap_runtime = runtime.clone();
7566            swap_runtime.start_requester = requester;
7567            let before = lock_snapshot(snapshot)
7568                .ok()
7569                .map(|state| state.spawn_generation);
7570            let end = swap::run_swap(
7571                spec,
7572                &swap_runtime,
7573                registry,
7574                process_liveness,
7575                snapshot,
7576                child,
7577                commands,
7578                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7579                reply,
7580            )
7581            .await;
7582            if before
7583                != lock_snapshot(snapshot)
7584                    .ok()
7585                    .map(|state| state.spawn_generation)
7586            {
7587                runtime.start_requester = swap_runtime.start_requester;
7588            }
7589            requeued.extend(end.requeue);
7590            true
7591        }
7592    }
7593}
7594
7595async fn restart_child(
7596    spec: &ModuleSpec,
7597    runtime: &SupervisorRuntimeConfig,
7598    registry: &Registry,
7599    process_liveness: &SupervisorProcessLiveness,
7600    snapshot: &SharedSnapshot,
7601    child: &mut Option<SupervisedChild>,
7602    drain_timeout: Duration,
7603) -> Result<(), SuperviseError> {
7604    // Restart cycles a running module; it must not silently start a disabled one.
7605    if !lock_snapshot(snapshot)?.enabled {
7606        return Err(SuperviseError::Disabled {
7607            module_id: spec.module_id.clone(),
7608        });
7609    }
7610    let stop_notice = begin_forwarding_drain_with_timeout(
7611        spec,
7612        runtime,
7613        registry,
7614        snapshot,
7615        None,
7616        RouteCloseReason::Restart,
7617        drain_timeout,
7618    )
7619    .await?;
7620
7621    if child.is_some() {
7622        drain_optional_child(
7623            &spec.module_id,
7624            spec.protocol,
7625            stop_notice,
7626            registry,
7627            runtime.forwarding.as_deref(),
7628            snapshot,
7629            &runtime.terminal_ring,
7630            &runtime.spawn_events,
7631            child,
7632            drain_timeout,
7633            ModuleState::Restarting,
7634            Some(true),
7635            &FailureContext::operator(),
7636        )
7637        .await?;
7638    } else {
7639        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7640            state.enabled = true;
7641            state.state = ModuleState::Restarting;
7642            clear_current_process_facts(state);
7643        })?;
7644        release_dead_registration(
7645            registry,
7646            runtime.forwarding.as_deref(),
7647            snapshot,
7648            &spec.module_id,
7649            &runtime.failure_context(false),
7650        )
7651        .await?;
7652    }
7653
7654    reset_restart_count(snapshot, &spec.module_id)?;
7655    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7656    schedule_respawn(
7657        runtime,
7658        snapshot,
7659        &spec.module_id,
7660        runtime.restart_policy.backoff,
7661        RespawnKind::Spawn,
7662        false,
7663    )
7664}
7665
7666async fn reload_child(
7667    spec: &ModuleSpec,
7668    runtime: &SupervisorRuntimeConfig,
7669    registry: &Registry,
7670    process_liveness: &SupervisorProcessLiveness,
7671    snapshot: &SharedSnapshot,
7672    child: &mut Option<SupervisedChild>,
7673) -> Result<(), SuperviseError> {
7674    // Reload cycles a running module; it must not silently start a disabled one.
7675    if !lock_snapshot(snapshot)?.enabled {
7676        return Err(SuperviseError::Disabled {
7677            module_id: spec.module_id.clone(),
7678        });
7679    }
7680    let stop_notice = begin_forwarding_drain(
7681        spec,
7682        runtime,
7683        registry,
7684        snapshot,
7685        Some(true),
7686        RouteCloseReason::Reload,
7687    )
7688    .await?;
7689
7690    if child.is_some() {
7691        drain_optional_child(
7692            &spec.module_id,
7693            spec.protocol,
7694            stop_notice,
7695            registry,
7696            runtime.forwarding.as_deref(),
7697            snapshot,
7698            &runtime.terminal_ring,
7699            &runtime.spawn_events,
7700            child,
7701            runtime.drain_timeout,
7702            ModuleState::Restarting,
7703            Some(true),
7704            &FailureContext::operator(),
7705        )
7706        .await?;
7707    } else {
7708        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7709            state.enabled = true;
7710            state.state = ModuleState::Restarting;
7711            clear_current_process_facts(state);
7712        })?;
7713        release_dead_registration(
7714            registry,
7715            runtime.forwarding.as_deref(),
7716            snapshot,
7717            &spec.module_id,
7718            &runtime.failure_context(false),
7719        )
7720        .await?;
7721    }
7722
7723    reset_restart_count(snapshot, &spec.module_id)?;
7724    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7725    schedule_respawn(
7726        runtime,
7727        snapshot,
7728        &spec.module_id,
7729        runtime.restart_policy.backoff,
7730        RespawnKind::Reload,
7731        false,
7732    )
7733}
7734
7735async fn finish_reload_child(
7736    spec: &ModuleSpec,
7737    runtime: &SupervisorRuntimeConfig,
7738    registry: &Registry,
7739    process_liveness: &SupervisorProcessLiveness,
7740    snapshot: &SharedSnapshot,
7741    child: &mut Option<SupervisedChild>,
7742) -> Result<(), SuperviseError> {
7743    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7744    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7745        Ok(next_child) => next_child,
7746        Err(err) => {
7747            return handle_reload_spawn_failure(
7748                spec,
7749                runtime,
7750                process_liveness,
7751                snapshot,
7752                child,
7753                format!("new child failed to spawn: {err}"),
7754            )
7755            .await;
7756        }
7757    };
7758    *child = Some(next_child);
7759
7760    let wait_outcome = {
7761        let active_child = child.as_mut().expect("new reload child was just stored");
7762        wait_for_registration_after_reload(
7763            registry,
7764            &spec.module_id,
7765            snapshot,
7766            active_child,
7767            REGISTRY_RELEASE_TIMEOUT,
7768        )
7769        .await?
7770    };
7771
7772    match wait_outcome {
7773        RegistrationWaitOutcome::Registered => {
7774            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7775            Ok(())
7776        }
7777        RegistrationWaitOutcome::Exited(exit_report) => {
7778            if let Some(active_child) = child.as_mut() {
7779                active_child.drain_stderr(&spec.module_id).await;
7780            }
7781            // Keep the reaped child's roster guard until its terminal is written.
7782            // Shutdown waits on that guard, not on the child Option used for respawn.
7783            let mut exited_child = child.take().expect("exited reload child is still stored");
7784            #[cfg(test)]
7785            if let Some(gate) = &runtime.test_reload_exit_record_gate {
7786                gate.reached.notify_one();
7787                gate.resume.notified().await;
7788            }
7789            let result = handle_reload_child_registration_failure(
7790                spec,
7791                runtime,
7792                registry,
7793                process_liveness,
7794                snapshot,
7795                child,
7796                ReloadRegistrationFailure {
7797                    exit_report: registration_failure_exit_report(exit_report),
7798                    reason: exited_child
7799                        .spawn_failure
7800                        .clone()
7801                        .unwrap_or_else(|| "new child exited before registering".to_string()),
7802                },
7803            )
7804            .await;
7805            exited_child.release_roster();
7806            result
7807        }
7808        RegistrationWaitOutcome::TimedOut => {
7809            let mut timed_out_child = child
7810                .take()
7811                .expect("timed-out reload child is still running");
7812            timed_out_child
7813                .start_kill()
7814                .map_err(|source| SuperviseError::Kill {
7815                    module_id: spec.module_id.clone(),
7816                    source,
7817                })?;
7818            let status = timed_out_child
7819                .wait()
7820                .await
7821                .map_err(|source| SuperviseError::Wait {
7822                    module_id: spec.module_id.clone(),
7823                    source,
7824                })?;
7825            timed_out_child.drain_stderr(&spec.module_id).await;
7826            handle_reload_child_registration_failure(
7827                spec,
7828                runtime,
7829                registry,
7830                process_liveness,
7831                snapshot,
7832                child,
7833                ReloadRegistrationFailure {
7834                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7835                        snapshot,
7836                        &timed_out_child,
7837                        &status,
7838                    )),
7839                    reason: format!(
7840                        "new child did not register within {:?}",
7841                        REGISTRY_RELEASE_TIMEOUT
7842                    ),
7843                },
7844            )
7845            .await
7846        }
7847    }
7848}
7849
7850#[allow(clippy::too_many_arguments)]
7851async fn set_child_enabled(
7852    spec: &ModuleSpec,
7853    runtime: &SupervisorRuntimeConfig,
7854    registry: &Registry,
7855    process_liveness: &SupervisorProcessLiveness,
7856    snapshot: &SharedSnapshot,
7857    child: &mut Option<SupervisedChild>,
7858    enabled: bool,
7859    failure: &FailureContext,
7860) -> Result<bool, SuperviseError> {
7861    let (current_enabled, current_state, respawn_pending) = {
7862        let state = lock_snapshot(snapshot)?;
7863        (state.enabled, state.state, state.respawn_pending)
7864    };
7865    // `start` (enable on an already-enabled module) heals TERMINAL states instead
7866    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
7867    // clean (Stopped) has no live process and no other in-band recovery — the
7868    // operator's start is the explicit recovery act and resets the budget. Without
7869    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
7870    // which the 2026-07-14 aft outage proved is a trap when the failed module is
7871    // the one providing every agent's shell.
7872    let revive_terminal = enabled
7873        && current_enabled
7874        && child.is_none()
7875        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7876            || (current_state == ModuleState::Restarting && !respawn_pending));
7877    if current_enabled == enabled && !revive_terminal {
7878        return Ok(false);
7879    }
7880
7881    if enabled {
7882        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7883            state.enabled = true;
7884            state.state = ModuleState::Starting;
7885            clear_current_process_facts(state);
7886        })?;
7887        #[cfg(test)]
7888        if runtime.test_seed_stale_facts_before_enable_spawn {
7889            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7890                state.process_alive = true;
7891                state.pid = Some(41);
7892                state.spawned_at_ms = Some(42);
7893                state.spawned_from = Some(PathBuf::from("/spawned/module"));
7894                state.spawned_file_identity = Some(SpawnedFileIdentity {
7895                    device: 43,
7896                    inode: 44,
7897                    #[cfg(windows)]
7898                    windows_image: None,
7899                });
7900            })?;
7901        }
7902        release_dead_registration(
7903            registry,
7904            runtime.forwarding.as_deref(),
7905            snapshot,
7906            &spec.module_id,
7907            failure,
7908        )
7909        .await?;
7910        reset_restart_count(snapshot, &spec.module_id)?;
7911        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7912        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7913            Ok(next_child) => next_child,
7914            Err(err) => {
7915                fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7916                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7917                return Err(err);
7918            }
7919        };
7920        *child = Some(next_child);
7921        debug!(module_id = %spec.module_id, "supervised module enabled");
7922        Ok(true)
7923    } else {
7924        let stop_notice = begin_forwarding_drain_if_configured(
7925            spec,
7926            runtime,
7927            registry,
7928            snapshot,
7929            Some(false),
7930            RouteCloseReason::Disable,
7931        )
7932        .await?;
7933        drain_optional_child(
7934            &spec.module_id,
7935            spec.protocol,
7936            stop_notice,
7937            registry,
7938            runtime.forwarding.as_deref(),
7939            snapshot,
7940            &runtime.terminal_ring,
7941            &runtime.spawn_events,
7942            child,
7943            runtime.drain_timeout,
7944            ModuleState::Disabled,
7945            Some(false),
7946            failure,
7947        )
7948        .await?;
7949        debug!(module_id = %spec.module_id, "supervised module disabled");
7950        Ok(true)
7951    }
7952}
7953
7954#[allow(clippy::too_many_arguments)]
7955async fn on_child_exit(
7956    spec: &ModuleSpec,
7957    policy: RestartPolicy,
7958    registry: &Registry,
7959    snapshot: &SharedSnapshot,
7960    terminal_ring: &Arc<Mutex<TerminalRing>>,
7961    spawn_events: &SpawnEventFeed,
7962    roster: &ChildRoster,
7963    exit_report: ExitReport,
7964    daemon_initiated: bool,
7965) -> NextAction {
7966    // Once the daemon has begun shutting down, no exit is a crash to recover
7967    // from: the module is exiting because the daemon is going away (EOF on its
7968    // connection, or a service manager signalling the whole cgroup). Record it
7969    // as such and never schedule a respawn, which would only start a process
7970    // for the shutdown to end again.
7971    if roster.is_closed() {
7972        return on_child_exit_during_daemon_shutdown(
7973            spec,
7974            registry,
7975            snapshot,
7976            terminal_ring,
7977            spawn_events,
7978            exit_report,
7979        )
7980        .await;
7981    }
7982    // Every stop the supervisor itself asks for (operator stop, disable,
7983    // restart, reload, swap, a health restart, a drain that runs out of budget)
7984    // takes the child out of the supervise loop and reaps it in
7985    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
7986    // that reaches this point was not requested by the daemon.
7987    //
7988    // For a subc-wire module a clean exit is still a stop: those modules are
7989    // written to re-raise SIGTERM, so a stray outside signal already reads as a
7990    // crash, and exiting 0 is a deliberate choice the module made. A
7991    // `protocol: "none"` module is a stock program we cannot change, and many
7992    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
7993    // stop would leave the module down for good after any stray signal, so it
7994    // goes through the crash path instead: it spends restart budget, respawns
7995    // with the crash backoff, and ends `failed` when the budget runs out.
7996    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7997        && running_protocol(spec, snapshot) == ModuleProtocol::None;
7998    match exit_report.kind {
7999        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8000            info!(
8001                module_id = %spec.module_id,
8002                exit_code = ?exit_report.code,
8003                exit_signal = ?exit_report.signal,
8004                "supervised module exited cleanly"
8005            );
8006            if let Err(err) = publish_after_terminal_record(
8007                &spec.module_id,
8008                || {
8009                    record_terminal(
8010                        &spec.module_id,
8011                        terminal_ring,
8012                        spawn_events,
8013                        &exit_report,
8014                        TerminalDisposition::Stopped,
8015                    )
8016                },
8017                || {
8018                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8019                        state.state = ModuleState::Stopped;
8020                        clear_current_process_facts(state);
8021                        state.last_exit = Some(exit_report.clone());
8022                    })
8023                },
8024            ) {
8025                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8026            }
8027            let registration_released = match wait_for_registration_release(
8028                registry,
8029                &spec.module_id,
8030                REGISTRY_RELEASE_TIMEOUT,
8031            )
8032            .await
8033            {
8034                Ok(()) => true,
8035                Err(err) => {
8036                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8037                    false
8038                }
8039            };
8040            NextAction::Stop {
8041                registration_released,
8042            }
8043        }
8044        ExitKind::Clean | ExitKind::Crash => {
8045            if unrequested_clean_exit_of_protocol_none {
8046                warn!(
8047                    module_id = %spec.module_id,
8048                    exit_code = ?exit_report.code,
8049                    exit_signal = ?exit_report.signal,
8050                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
8051                );
8052            } else {
8053                warn!(
8054                    module_id = %spec.module_id,
8055                    exit_code = ?exit_report.code,
8056                    exit_signal = ?exit_report.signal,
8057                    "supervised module exited abnormally (crash)"
8058                );
8059            }
8060            let mut restart_schedule = None;
8061            let mut disposition = TerminalDisposition::Disabled;
8062            let mut final_state = ModuleState::Disabled;
8063            // Set only when the budget is what stopped the module, so the
8064            // terminal record says which limit was hit rather than leaving
8065            // `failed` to be read as "crashed once, badly".
8066            let mut disposition_detail = lock_snapshot(snapshot)
8067                .ok()
8068                .and_then(|mut state| state.spawn_failure.take());
8069            let now = Instant::now();
8070            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8071                if state.enabled {
8072                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
8073                        // Reserving the last slot exhausts the budget for the
8074                        // replacement, not for this exit. Publish its scheduled
8075                        // recovery atomically with the reservation so connection
8076                        // cleanup sees it while terminal recording still holds
8077                        // the visible state at Running.
8078                        state.respawn_pending = true;
8079                        final_state = ModuleState::Restarting;
8080                        restart_schedule = Some(schedule);
8081                        disposition = TerminalDisposition::Restarting;
8082                    } else {
8083                        final_state = ModuleState::Failed;
8084                        disposition = TerminalDisposition::Failed;
8085                        let budget = policy.budget_exhausted_detail();
8086                        disposition_detail =
8087                            Some(disposition_detail.take().map_or_else(
8088                                || budget.clone(),
8089                                |cause| format!("{cause}; {budget}"),
8090                            ));
8091                    }
8092                } else {
8093                    disposition = TerminalDisposition::Disabled;
8094                }
8095            }) {
8096                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8097                return NextAction::Stop {
8098                    registration_released: false,
8099                };
8100            }
8101            if disposition == TerminalDisposition::Failed {
8102                // The window is in the message, not only in the fields: this line
8103                // is read in a scrollback where a bare `max_restarts=3` reads as a
8104                // lifetime cap and sends the operator looking for three crashes
8105                // that never happened together.
8106                error!(
8107                    module_id = %spec.module_id,
8108                    max_restarts = policy.max_restarts,
8109                    window_secs = policy.window.as_secs(),
8110                    "module stopped: {}",
8111                    policy.budget_exhausted_detail()
8112                );
8113            }
8114            if let Err(err) = publish_after_terminal_record(
8115                &spec.module_id,
8116                || {
8117                    record_terminal_with_detail(
8118                        &spec.module_id,
8119                        terminal_ring,
8120                        spawn_events,
8121                        &exit_report,
8122                        disposition,
8123                        disposition_detail,
8124                    )
8125                },
8126                || {
8127                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8128                        state.state = final_state;
8129                        clear_current_process_facts(state);
8130                        if final_state == ModuleState::Failed
8131                            && daemon_initiated
8132                            && !roster.is_closed()
8133                        {
8134                            state.failed_retry.schedule(&spec.module_id);
8135                        }
8136                        state.last_exit = Some(exit_report.clone());
8137                    })
8138                },
8139            ) {
8140                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8141            }
8142
8143            if let Some(schedule) = restart_schedule {
8144                NextAction::Restart {
8145                    schedule: Some(schedule),
8146                }
8147            } else {
8148                let registration_released = match wait_for_registration_release(
8149                    registry,
8150                    &spec.module_id,
8151                    REGISTRY_RELEASE_TIMEOUT,
8152                )
8153                .await
8154                {
8155                    Ok(()) => true,
8156                    Err(err) => {
8157                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8158                        false
8159                    }
8160                };
8161                NextAction::Stop {
8162                    registration_released,
8163                }
8164            }
8165        }
8166        ExitKind::DeliberateSeverance => {
8167            warn!(
8168                module_id = %spec.module_id,
8169                exit_code = ?exit_report.code,
8170                exit_signal = ?exit_report.signal,
8171                "supervised module exited after deliberate connection severance"
8172            );
8173            let should_restart = match lock_snapshot(snapshot) {
8174                Ok(state) => state.enabled,
8175                Err(err) => {
8176                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8177                    return NextAction::Stop {
8178                        registration_released: false,
8179                    };
8180                }
8181            };
8182            let final_state = if should_restart {
8183                ModuleState::Restarting
8184            } else {
8185                ModuleState::Disabled
8186            };
8187            if let Err(err) = publish_after_terminal_record(
8188                &spec.module_id,
8189                || {
8190                    record_terminal(
8191                        &spec.module_id,
8192                        terminal_ring,
8193                        spawn_events,
8194                        &exit_report,
8195                        terminal_disposition(final_state),
8196                    )
8197                },
8198                || {
8199                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8200                        state.state = final_state;
8201                        clear_current_process_facts(state);
8202                        state.last_exit = Some(exit_report.clone());
8203                        state.lifetime_restarts += 1;
8204                    })
8205                },
8206            ) {
8207                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8208                return NextAction::Stop {
8209                    registration_released: false,
8210                };
8211            }
8212
8213            if should_restart {
8214                NextAction::Restart { schedule: None }
8215            } else {
8216                let registration_released = match wait_for_registration_release(
8217                    registry,
8218                    &spec.module_id,
8219                    REGISTRY_RELEASE_TIMEOUT,
8220                )
8221                .await
8222                {
8223                    Ok(()) => true,
8224                    Err(err) => {
8225                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8226                        false
8227                    }
8228                };
8229                NextAction::Stop {
8230                    registration_released,
8231                }
8232            }
8233        }
8234    }
8235}
8236
8237async fn on_child_exit_during_daemon_shutdown(
8238    spec: &ModuleSpec,
8239    registry: &Registry,
8240    snapshot: &SharedSnapshot,
8241    terminal_ring: &Arc<Mutex<TerminalRing>>,
8242    spawn_events: &SpawnEventFeed,
8243    exit_report: ExitReport,
8244) -> NextAction {
8245    info!(
8246        module_id = %spec.module_id,
8247        exit_code = ?exit_report.code,
8248        exit_signal = ?exit_report.signal,
8249        exit_kind = ?exit_report.kind,
8250        "supervised module exited during daemon shutdown; not restarting it"
8251    );
8252    if let Err(err) = publish_after_terminal_record(
8253        &spec.module_id,
8254        || {
8255            record_terminal(
8256                &spec.module_id,
8257                terminal_ring,
8258                spawn_events,
8259                &exit_report,
8260                TerminalDisposition::DaemonShutdown,
8261            )
8262        },
8263        || {
8264            update_snapshot(snapshot, Some(&spec.module_id), |state| {
8265                state.state = ModuleState::Stopped;
8266                clear_current_process_facts(state);
8267                state.last_exit = Some(exit_report.clone());
8268            })
8269        },
8270    ) {
8271        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8272    }
8273    let registration_released =
8274        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8275            .await
8276            .is_ok();
8277    NextAction::Stop {
8278        registration_released,
8279    }
8280}
8281
8282// Keep the status lock out of terminal I/O. Readers may observe the previous
8283// state while recording is in progress, but never a completed exit without its
8284// history. Journal errors are logged by the journal and retained in the ring;
8285// a recording panic still publishes the final state before resuming the unwind.
8286fn publish_after_terminal_record<T>(
8287    module_id: &str,
8288    record: impl FnOnce(),
8289    publish: impl FnOnce() -> T,
8290) -> T {
8291    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8292    let published = publish();
8293    if let Err(panic) = recorded {
8294        error!(
8295            module_id,
8296            "terminal recording panicked; published final state before resuming unwind"
8297        );
8298        std::panic::resume_unwind(panic);
8299    }
8300    published
8301}
8302
8303fn fail_after_wait_error(
8304    module_id: &str,
8305    snapshot: &SharedSnapshot,
8306    terminal_ring: &Arc<Mutex<TerminalRing>>,
8307    spawn_events: &SpawnEventFeed,
8308    detail: String,
8309    failure: &FailureContext,
8310) {
8311    let report = wait_error_exit_report();
8312    publish_after_terminal_record(
8313        module_id,
8314        || {
8315            record_terminal_with_detail(
8316                module_id,
8317                terminal_ring,
8318                spawn_events,
8319                &report,
8320                TerminalDisposition::Failed,
8321                Some(detail),
8322            )
8323        },
8324        || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8325    );
8326}
8327
8328#[cfg(test)]
8329fn record_wait_error_terminal(
8330    module_id: &str,
8331    terminal_ring: &Arc<Mutex<TerminalRing>>,
8332    spawn_events: &SpawnEventFeed,
8333) {
8334    record_terminal(
8335        module_id,
8336        terminal_ring,
8337        spawn_events,
8338        &wait_error_exit_report(),
8339        TerminalDisposition::Failed,
8340    );
8341}
8342
8343fn record_terminal(
8344    module_id: &str,
8345    terminal_ring: &Arc<Mutex<TerminalRing>>,
8346    spawn_events: &SpawnEventFeed,
8347    exit_report: &ExitReport,
8348    disposition: TerminalDisposition,
8349) {
8350    record_terminal_with_detail(
8351        module_id,
8352        terminal_ring,
8353        spawn_events,
8354        exit_report,
8355        disposition,
8356        None,
8357    );
8358}
8359
8360/// The ring lock is held only to capture the read (see
8361/// `TerminalJournal::capture_read`), so this module's exits keep recording
8362/// while the journal files are read. Blocking: it reads files.
8363fn durable_terminal_history_of(
8364    terminal_ring: &Mutex<TerminalRing>,
8365    module_id: &str,
8366) -> subc_control::TerminalHistory {
8367    let read = terminal_ring
8368        .lock()
8369        .unwrap_or_else(|p| p.into_inner())
8370        .capture_durable_history();
8371    read.read(module_id)
8372}
8373
8374fn record_terminal_with_detail(
8375    module_id: &str,
8376    terminal_ring: &Arc<Mutex<TerminalRing>>,
8377    spawn_events: &SpawnEventFeed,
8378    exit_report: &ExitReport,
8379    disposition: TerminalDisposition,
8380    disposition_detail: Option<String>,
8381) {
8382    #[cfg(test)]
8383    terminal_history_tests::before_terminal_record(spawn_events);
8384    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8385    let record = TerminalRecord {
8386        exit_code: exit_report.code,
8387        exit_signal: exit_report.signal,
8388        at_ms: exit_report.at_ms,
8389        disposition,
8390        exit_kind: exit_report.kind.into(),
8391        disposition_detail,
8392    };
8393    terminal_ring
8394        .lock()
8395        .unwrap_or_else(|poisoned| poisoned.into_inner())
8396        .record_exit(module_id, record);
8397}
8398
8399fn untrack_if_registration_released(
8400    process_liveness: &SupervisorProcessLiveness,
8401    registry: &Registry,
8402    module_id: &str,
8403    snapshot: &SharedSnapshot,
8404) {
8405    match registry.get_module(module_id) {
8406        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8407        Ok(Some(_)) => {}
8408        Err(err) => {
8409            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8410        }
8411    }
8412}
8413
8414/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
8415/// then apply the module's configured entries minus daemon-private capture keys.
8416///
8417/// Separated from `spawn_child` only so it can be asserted without spawning a
8418/// process — a duplicate of this logic in a test would pass while the real one
8419/// drifted, which is the defect class this function exists to avoid.
8420/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
8421/// nonce. A `protocol: "none"` module gets neither, because it cannot use
8422/// either and the argument would stop a stock binary from starting at all.
8423/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
8424///
8425/// The plain-spawn form, kept for the tests that assert its plan; spawns go
8426/// through [`apply_wire_spawn_args_for_role`].
8427#[cfg(test)]
8428fn apply_wire_spawn_args(
8429    command: &mut Command,
8430    spec: &ModuleSpec,
8431    connection_file_path: Option<&std::path::Path>,
8432    handle: Option<&SupervisorHandle>,
8433) -> Result<Option<NonceHandoff>, SuperviseError> {
8434    apply_wire_spawn_args_for_role(
8435        command,
8436        spec,
8437        connection_file_path,
8438        handle,
8439        SpawnRole::Plain,
8440    )
8441}
8442
8443/// The read end of a spawn's launch-nonce pipe, prepared by
8444/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
8445/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
8446/// server before spawn and authenticates its client after the child exists.
8447#[cfg(unix)]
8448type NonceHandoff = subc_os::LaunchNonceHandoff;
8449#[cfg(windows)]
8450type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8451#[cfg(not(any(unix, windows)))]
8452type NonceHandoff = std::convert::Infallible;
8453
8454/// Create the launch nonce that a new module process must present when it
8455/// registers or opens a consumer connection, including a replacement process
8456/// started alongside a still-serving module.
8457///
8458/// A plain spawn replaces the module's recorded nonce. A swap candidate records
8459/// a separate candidate token so the still-serving incumbent and its consumers
8460/// keep their nonce. Both records are installed before the process exists, so
8461/// the child's initial HELLO registration cannot arrive ahead of its nonce.
8462///
8463/// On Unix the nonce is delivered only through a pipe. It is written into
8464/// a pipe whose read end the child gets as descriptor 3, named by
8465/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
8466/// process of the same user cannot read it with `ps eww`. That handoff is
8467/// returned rather than installed here, because installing it replaces
8468/// whatever the child has at descriptor 3 and so must be the last pre-exec
8469/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
8470/// the Linux callback that writes the process into its cgroup. Only after that
8471/// does `handoff.install_last` register the descriptor-3 replacement, so it
8472/// cannot close a descriptor the cgroup callback still needs.
8473/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
8474fn apply_wire_spawn_args_for_role(
8475    command: &mut Command,
8476    spec: &ModuleSpec,
8477    connection_file_path: Option<&std::path::Path>,
8478    handle: Option<&SupervisorHandle>,
8479    role: SpawnRole,
8480) -> Result<Option<NonceHandoff>, SuperviseError> {
8481    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8482    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
8483    // included: a daemon started from a module's process tree inherits it,
8484    // and passing it on would point the child at a descriptor it does not
8485    // have.
8486    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8487    #[cfg(windows)]
8488    {
8489        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8490        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8491    }
8492    // Remove inherited or configured copies too: withholding must mean absent.
8493    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8494    if spec.protocol == ModuleProtocol::None {
8495        return Ok(None);
8496    }
8497    if let Some(connection_file_path) = connection_file_path {
8498        command.arg(SUBC_ARG).arg(connection_file_path);
8499    }
8500
8501    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
8502    // route.open attestation. Reserved modules additionally use the same nonce
8503    // for HELLO id-squatting protection. A respawn rotates both records.
8504    let nonce = generate_launch_nonce()?;
8505    if let Some(handle) = handle {
8506        match role {
8507            SpawnRole::Plain => {
8508                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8509                if spec.reserved {
8510                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8511                }
8512            }
8513            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8514        }
8515    }
8516    #[cfg(unix)]
8517    let handoff = {
8518        let handoff =
8519            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8520                program: spec.program.clone(),
8521                source,
8522                cgroup_path: None,
8523            })?;
8524        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8525        Some(handoff)
8526    };
8527    #[cfg(windows)]
8528    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8529    #[cfg(not(any(unix, windows)))]
8530    let handoff = None;
8531    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
8532    // until the daemon's per-spawn delivery records show that every running
8533    // Windows module has read its secret from the named pipe.
8534    #[cfg(not(unix))]
8535    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8536    Ok(handoff)
8537}
8538
8539fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8540    command.env_remove(CK_LOG_ENV);
8541    // The spawn role is the supervisor's to set, and only on a swap candidate
8542    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8543    // it, is what makes it absent on a plain spawn: the daemon's own
8544    // environment could carry it, and so could a spec built outside daemon
8545    // config (config refuses it as an `env` key). A module reading it on a
8546    // plain restart would pick the long swap budget and leave callers waiting.
8547    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8548    for (key, value) in &spec.env {
8549        // cortexkit-log currently exposes retention only as a Rust struct, not
8550        // environment names. These values are daemon-private sink metadata and
8551        // must never become a public child-process contract by being inherited.
8552        if matches!(
8553            key.as_str(),
8554            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8555        ) || key == SUBC_SPAWN_ROLE_ENV
8556        {
8557            continue;
8558        }
8559        command.env(key, value);
8560    }
8561}
8562
8563#[cfg(windows)]
8564fn prepare_windows_nonce_pipe(
8565    command: &mut Command,
8566    spec: &ModuleSpec,
8567    nonce: &str,
8568) -> Option<NonceHandoff> {
8569    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8570        Ok(handoff) => {
8571            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8572            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8573            // still register until every module reads the pipe. A reader without
8574            // this variable set to env returns the pipe error instead of using
8575            // an inherited or configured environment copy.
8576            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8577            Some(handoff)
8578        }
8579        Err(error) => {
8580            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8581            None
8582        }
8583    }
8584}
8585
8586#[cfg(windows)]
8587fn start_windows_nonce_pipe(
8588    handoff: Option<NonceHandoff>,
8589    pid: u32,
8590    spec: &ModuleSpec,
8591) -> Option<subc_os::LaunchNoncePipeDelivery> {
8592    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8593    // same maximum wait used for registration during a reload. Older modules
8594    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8595    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8596        Ok(delivery) => Some(delivery),
8597        Err(error) => {
8598            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8599            None
8600        }
8601    })
8602}
8603
8604fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8605    #[cfg(windows)]
8606    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8607        return Some(
8608            if snapshot
8609                .pipe_consumed
8610                .as_ref()
8611                .is_some_and(|consumed| consumed.get().is_some())
8612            {
8613                "pipe"
8614            } else {
8615                "env"
8616            }
8617            .to_owned(),
8618        );
8619    }
8620    let _ = snapshot;
8621    None
8622}
8623
8624/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8625/// of a blue/green swap.
8626#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8627enum SpawnRole {
8628    Plain,
8629    SwapCandidate,
8630}
8631
8632/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8633/// `apply_child_env` has already removed the variable for every spawn.
8634fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8635    if role == SpawnRole::SwapCandidate {
8636        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8637    }
8638}
8639
8640fn previous_instance_error(
8641    spec: &ModuleSpec,
8642    pid: u32,
8643    reason: crate::live_children::RetainedReason,
8644) -> SuperviseError {
8645    SuperviseError::Spawn {
8646        program: spec.program.clone(),
8647        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8648        cgroup_path: None,
8649    }
8650}
8651
8652fn spawn_child(
8653    spec: &ModuleSpec,
8654    connection_file_path: Option<&std::path::Path>,
8655    handle: Option<&SupervisorHandle>,
8656    ring: &Arc<Mutex<StderrRing>>,
8657    capture_logs_dir: Option<&std::path::Path>,
8658    roster: &ChildRoster,
8659    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8660) -> Result<SupervisedChild, SuperviseError> {
8661    spawn_child_in_slot(
8662        spec,
8663        connection_file_path,
8664        handle,
8665        ring,
8666        capture_logs_dir,
8667        roster,
8668        #[cfg(target_os = "linux")]
8669        cgroup_placement,
8670        SpawnRole::Plain,
8671        false,
8672    )
8673}
8674
8675/// Spawn one process of `spec` into a slot.
8676///
8677/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
8678/// A swap candidate needs a different cgroup from the process it is replacing,
8679/// which is still alive: in the same cgroup the two would be one kill domain,
8680/// and killing a failed candidate could take the incumbent with it.
8681///
8682/// The stderr capture file is `<module_id>.stderr.log` for every process of
8683/// the module, whichever slot it is in, because that is the one file
8684/// `ck module logs` reads. During a swap's overlap both processes append to it;
8685/// the daemon writes whole lines, so the two interleave by line, which is also
8686/// the merged view an operator wants while a swap runs.
8687#[allow(clippy::too_many_arguments)]
8688fn spawn_child_in_slot(
8689    spec: &ModuleSpec,
8690    connection_file_path: Option<&std::path::Path>,
8691    handle: Option<&SupervisorHandle>,
8692    ring: &Arc<Mutex<StderrRing>>,
8693    capture_logs_dir: Option<&std::path::Path>,
8694    roster: &ChildRoster,
8695    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8696    role: SpawnRole,
8697    alternate_slot: bool,
8698) -> Result<SupervisedChild, SuperviseError> {
8699    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8700        return Err(previous_instance_error(spec, pid, reason));
8701    }
8702    if roster.is_closed() {
8703        return Err(SuperviseError::Spawn {
8704            program: spec.program.clone(),
8705            source: io::Error::other("the daemon is shutting down; not starting a new process"),
8706            cgroup_path: None,
8707        });
8708    }
8709    #[cfg(target_os = "linux")]
8710    let cgroup_name = {
8711        // Slot names alone are not kill domains: a retired incumbent may still
8712        // be draining when a later enable/restart spawns into the same slot.
8713        // Decimal entropy keeps the suffix unambiguous; Placement performs
8714        // the module-id escaping and constructs the filesystem path.
8715        if cgroup_placement.is_none() {
8716            swap::cgroup_name(&spec.module_id, alternate_slot)
8717        } else {
8718            let nonce = generate_launch_nonce()?;
8719            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8720            // Leave room for byte escaping and the suffix under NAME_MAX. The
8721            // label is only for humans; the nonce identifies the kill domain.
8722            let mut end = spec.module_id.len().min(64);
8723            while !spec.module_id.is_char_boundary(end) {
8724                end -= 1;
8725            }
8726            format!(
8727                "{}_{suffix}",
8728                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8729            )
8730        }
8731    };
8732    #[cfg(not(target_os = "linux"))]
8733    let _ = alternate_slot;
8734    #[cfg(target_os = "macos")]
8735    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8736    #[cfg(windows)]
8737    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8738        program: spec.program.clone(),
8739        source,
8740        cgroup_path: None,
8741    })?;
8742    #[cfg(windows)]
8743    let mut command = Command::new(image_capture.path());
8744    #[cfg(not(any(target_os = "macos", windows)))]
8745    let mut command = Command::new(&spec.program);
8746    command.args(&spec.args);
8747    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
8748    // that is the whole of the intent, so remove that one key rather than the
8749    // environment.
8750    //
8751    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
8752    // and took the POSIX environment with it. Modules spawned that way had no
8753    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
8754    // logging:
8755    //
8756    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
8757    //     both unset it fell back to the temp dir alone and `ck` could not find
8758    //     a daemon running on the same machine from inside any module's process
8759    //     tree — reporting a path the file has never lived at, which reads as
8760    //     "the daemon did not write its file".
8761    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
8762    //     the RELATIVE `.local/share`, so a module deriving its own store path
8763    //     resolved it against its own CWD. That is the store-fragmentation
8764    //     defect the daemon already refuses in config (`parse_doc` rejects a
8765    //     relative `storage.data_home`) arriving by derivation instead.
8766    //   * anything a module spawns inherited it: git without ~/.gitconfig,
8767    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
8768    //     quietly rather than erroring.
8769    //
8770    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
8771    // offered one candidate under /tmp while the file sat in /run/user/1000.
8772    //
8773    // A configured module is unaffected either way: `module_spec()` puts the
8774    // resolved CK_LOG into `spec.env`, which is applied below and therefore
8775    // wins over anything ambient.
8776    apply_child_env(&mut command, spec);
8777    apply_spawn_role(&mut command, role);
8778    let nonce_handoff =
8779        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8780
8781    #[cfg(target_os = "linux")]
8782    let cgroup_path = cgroup_placement
8783        .map(|placement| placement.module_path(&cgroup_name))
8784        .transpose()
8785        .map_err(|source| SuperviseError::Cgroup {
8786            module_id: spec.module_id.clone(),
8787            source,
8788        })?;
8789    #[cfg(not(target_os = "linux"))]
8790    let cgroup_path: Option<PathBuf> = None;
8791    #[cfg(target_os = "linux")]
8792    if let Some(path) = &cgroup_path {
8793        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8794            if let Some(placement) = cgroup_placement {
8795                remove_module_cgroup(placement, &cgroup_name);
8796            }
8797            return Err(error);
8798        }
8799    }
8800
8801    let output_sink = if let Some(logs_dir) = capture_logs_dir {
8802        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8803        match ChildOutputSink::open(&path, capture_retention(spec)) {
8804            Ok(sink) => sink,
8805            Err(error) => {
8806                warn!(
8807                    module_id = %spec.module_id,
8808                    path = %path.display(),
8809                    error = %error,
8810                    "could not open child output capture file; forwarding to stderr"
8811                );
8812                ChildOutputSink::Stderr
8813            }
8814        }
8815    } else {
8816        ChildOutputSink::Stderr
8817    };
8818
8819    command.stdout(Stdio::piped());
8820    command.stderr(Stdio::piped());
8821    command.kill_on_drop(true);
8822    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
8823    // before exec). In the daemon's group, a service manager that kills the
8824    // job's process group when the daemon exits (launchd's default) killed
8825    // every module at the same moment its control connection closed, so no
8826    // module ever ran its EOF teardown on a daemon stop. Outside that group a
8827    // module is reached only by the daemon: the EOF it sees when its
8828    // connection closes, and the bounded stop in `child_roster` for anything
8829    // still running after that. On Linux this composes with the cgroup
8830    // placement above: that is a pre_exec write to cgroup.procs, std performs
8831    // setpgid in the child before running pre_exec callbacks, and the two
8832    // change independent process attributes.
8833    //
8834    // stdin is /dev/null because a process outside the terminal's foreground
8835    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
8836    // by hand would otherwise hand down. Under a service manager stdin is
8837    // already /dev/null.
8838    #[cfg(unix)]
8839    command.process_group(0);
8840    command.stdin(Stdio::null());
8841    // The LAST pre-exec step, after the cgroup placement above: installing the
8842    // nonce at descriptor 3 replaces whatever the child had there, which could
8843    // be the descriptor an earlier step writes through.
8844    #[cfg(unix)]
8845    if let Some(handoff) = nonce_handoff {
8846        handoff.install_last(command.as_std_mut());
8847    }
8848    #[cfg(not(any(unix, windows)))]
8849    let _ = nonce_handoff;
8850
8851    // Containment, step 1 of 3 (issue #109): create the child suspended so it
8852    // cannot run a single instruction -- and therefore cannot spawn a
8853    // grandchild -- before it is in the job. See `contain_spawned_child` for the
8854    // other two steps and why the window matters.
8855    #[cfg(windows)]
8856    {
8857        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8858            program: spec.program.clone(),
8859            source,
8860            cgroup_path: None,
8861        })?;
8862        subc_jobobject::suspend_on_create_async(&mut command);
8863    }
8864    // Timestamp before the spawn call: on Unix the child can serve before that
8865    // call even returns. This conservative bound never labels an older child
8866    // as a replacement for a request received during process creation.
8867    let spawned_at = Instant::now();
8868    let mut child = match command.spawn() {
8869        Ok(child) => child,
8870        Err(source) => {
8871            #[cfg(target_os = "linux")]
8872            if let Some(placement) = cgroup_placement {
8873                remove_module_cgroup(placement, &cgroup_name);
8874            }
8875            return Err(SuperviseError::Spawn {
8876                program: spec.program.clone(),
8877                source,
8878                cgroup_path,
8879            });
8880        }
8881    };
8882    #[cfg(test)]
8883    spawn_record_tests::after_create(&child, spec);
8884    // The parent must close its writer now: the acknowledgement pipe reports EOF
8885    // only when every writer is gone, and the child's copy closes when the
8886    // trampoline replaces itself with the module. Command holds only an integer
8887    // in its pre_exec callback, not another writer.
8888    #[cfg(target_os = "macos")]
8889    drop(exec_ack);
8890
8891    // Bind while the child is still suspended, before job assignment resumes it.
8892    // Keeping the executable handle open across CreateProcessW denies rename
8893    // and deletion, so the file cannot be exchanged during spawn. Bind that
8894    // file ID to the process handle returned by spawn, never a reopened PID.
8895    #[cfg(windows)]
8896    let windows_image = match image_capture.bind_tokio(&child) {
8897        Ok(image) => Arc::new(image),
8898        Err(source) => {
8899            kill_and_reap_rejected_child(&mut child);
8900            return Err(SuperviseError::Spawn {
8901                program: spec.program.clone(),
8902                source,
8903                cgroup_path: None,
8904            });
8905        }
8906    };
8907    // Assign the suspended child to its module job; resume only after its
8908    // process identity has been durably recorded for orphan recovery.
8909    #[cfg(windows)]
8910    let job = contain_spawned_child(&mut child, spec)?;
8911    let spawned_at_ms = unix_ms_now();
8912    #[cfg(not(windows))]
8913    let spawned_from = spec.program.clone();
8914    #[cfg(windows)]
8915    let spawned_from = windows_image.path().to_path_buf();
8916    #[cfg(not(windows))]
8917    let spawned_file_identity = spawned_file_identity(&spawned_from);
8918    #[cfg(windows)]
8919    let spawned_file_identity = Some(SpawnedFileIdentity {
8920        device: 0,
8921        inode: 0,
8922        windows_image: Some(Arc::clone(&windows_image)),
8923    });
8924    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8925        program: spec.program.clone(),
8926        source: io::Error::other("spawned child exposed no live pid"),
8927        cgroup_path: cgroup_path.clone(),
8928    })?;
8929    #[cfg(windows)]
8930    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8931    #[cfg(not(windows))]
8932    let process_start_time = crate::provenance::process_start_time(pid);
8933    #[cfg(windows)]
8934    let process_start_time = Some(windows_image.start_time());
8935    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8936    #[cfg(all(test, target_os = "macos"))]
8937    privacy_exec_boundary_tests::before_image_sample(spec, pid);
8938    // Record the committed kernel image, not the configured pathname: PATH
8939    // lookup and shebang interpretation can select a different executable.
8940    // Linux's posix_spawn can return before that image is installed, so the
8941    // observer also waits for the kernel's first-exec flag to clear.
8942    #[cfg(not(windows))]
8943    let recorded_image = observe_spawned_image(pid);
8944    // spawn() confirms only the first exec, into the trampoline. Never persist
8945    // the trampoline image; the asynchronous acknowledgement publishes the
8946    // module image once the trampoline has replaced itself with the module.
8947    #[cfg(target_os = "macos")]
8948    let recorded_image = if privacy_exec.is_some() {
8949        None
8950    } else {
8951        recorded_image
8952    };
8953    #[cfg(target_os = "linux")]
8954    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8955    #[cfg(not(target_os = "linux"))]
8956    let recorded_cgroup_name = None;
8957    let roster_guard = roster.admit(
8958        spec.module_id.clone(),
8959        pid,
8960        spec.protocol,
8961        process_start_time,
8962        crate::child_roster::RecordedIdentity {
8963            #[cfg(not(windows))]
8964            start_time: recorded_image.map(|image| image.start_time),
8965            #[cfg(windows)]
8966            start_time: process_start_time,
8967            #[cfg(not(windows))]
8968            executable: recorded_image
8969                .and_then(|image| image.executable)
8970                .map(crate::live_children::ExecutableIdentity::from),
8971            #[cfg(windows)]
8972            executable: Some(crate::live_children::ExecutableIdentity::from(
8973                windows_image.identity(),
8974            )),
8975            cgroup_name: recorded_cgroup_name,
8976            #[cfg(target_os = "linux")]
8977            cgroup_placement: cgroup_placement.cloned(),
8978        },
8979    );
8980    let roster_guard = match roster_guard {
8981        Ok(guard) => guard,
8982        Err(failure) => {
8983            return failure.with_cleanup(|source| {
8984                #[cfg(windows)]
8985                if let Some(job) = &job {
8986                    let _ = job.terminate();
8987                }
8988                #[cfg(target_os = "linux")]
8989                kill_module_cgroup(cgroup_placement, &cgroup_name);
8990                kill_and_reap_rejected_child(&mut child);
8991                #[cfg(target_os = "linux")]
8992                if let Some(placement) = cgroup_placement {
8993                    remove_module_cgroup(placement, &cgroup_name);
8994                }
8995                Err(SuperviseError::Spawn {
8996                    program: spec.program.clone(),
8997                    source: io::Error::other(format!(
8998                        "could not durably record module child: {source}"
8999                    )),
9000                    cgroup_path,
9001                })
9002            });
9003        }
9004    };
9005    // The pre-spawn check can pass just before shutdown begins. Ownership is
9006    // published at the start of admission, even while a record write is queued.
9007    // If shutdown already saw an empty roster before that publication, this
9008    // check sees the roster closed and ends the child before starting a module.
9009    if roster.is_closed() {
9010        // This child was never admitted, so there is no module protocol shutdown to wait for.
9011        #[cfg(target_os = "linux")]
9012        kill_module_cgroup(cgroup_placement, &cgroup_name);
9013        if let Err(error) = child.start_kill() {
9014            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9015        }
9016        #[cfg(target_os = "linux")]
9017        if let Some(placement) = cgroup_placement {
9018            // This spawn was never admitted, so shutdown has no roster entry
9019            // to await. Do not detach its cleanup: the runtime could exit
9020            // before that task reaps the rejected child and removes its group.
9021            while matches!(child.try_wait(), Ok(None)) {
9022                std::thread::yield_now();
9023            }
9024            if matches!(
9025                subc_cgroup::kill_module(Some(placement), &cgroup_name),
9026                subc_cgroup::KillOutcome::Killed
9027            ) {
9028                if let Ok(path) = placement.module_path(&cgroup_name) {
9029                    while std::fs::read_to_string(path.join("cgroup.events"))
9030                        .ok()
9031                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9032                    {
9033                        std::thread::yield_now();
9034                    }
9035                }
9036            }
9037            remove_module_cgroup(placement, &cgroup_name);
9038        }
9039        drop(roster_guard);
9040        return Err(SuperviseError::Spawn {
9041            program: spec.program.clone(),
9042            source: io::Error::other(
9043                "the daemon began shutting down while this process was starting; ended it",
9044            ),
9045            cgroup_path,
9046        });
9047    }
9048
9049    #[cfg(windows)]
9050    if let Err(error) = resume_suspended_child(pid, spec) {
9051        kill_and_reap_rejected_child(&mut child);
9052        return Err(error);
9053    }
9054
9055    let stdout_pump = match child.stdout.take() {
9056        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9057        None => {
9058            warn!(
9059                module_id = %spec.module_id,
9060                "spawned child exposed no stdout pipe; file capture will be incomplete"
9061            );
9062            None
9063        }
9064    };
9065    let stderr_pump = match child.stderr.take() {
9066        Some(stderr) => {
9067            let generation = ring
9068                .lock()
9069                .unwrap_or_else(|poisoned| poisoned.into_inner())
9070                .begin_process();
9071            Some(StderrPump {
9072                task: tokio::spawn(pump_stderr_to(
9073                    stderr,
9074                    Arc::clone(ring),
9075                    generation,
9076                    output_sink,
9077                )),
9078                generation,
9079            })
9080        }
9081        None => {
9082            // Spawning succeeded but the pipe did not materialise. Recording it as
9083            // uncaptured keeps the tail honest: the alternative is an empty tail
9084            // that reads as a module which printed nothing.
9085            ring.lock()
9086                .unwrap_or_else(|poisoned| poisoned.into_inner())
9087                .mark_not_captured("stderr pipe was not available on spawn");
9088            warn!(
9089                module_id = %spec.module_id,
9090                "spawned child exposed no stderr pipe; tail will be unavailable"
9091            );
9092            None
9093        }
9094    };
9095
9096    let exec = Arc::new(crate::activity::ExecOutcome::new());
9097    #[cfg(target_os = "macos")]
9098    let pending_exec = privacy_exec.is_some();
9099    #[cfg(not(target_os = "macos"))]
9100    let pending_exec = false;
9101    if !pending_exec {
9102        exec.finish(crate::activity::ExecState::CommittedExec);
9103    }
9104    Ok(SupervisedChild {
9105        child,
9106        exec: crate::activity::ExecOwner(exec),
9107        #[cfg(windows)]
9108        nonce_delivery,
9109        protocol: spec.protocol,
9110        #[cfg(target_os = "linux")]
9111        module_id: cgroup_name,
9112        #[cfg(target_os = "linux")]
9113        cgroup_placement: cgroup_placement.cloned(),
9114        #[cfg(windows)]
9115        job,
9116        stdout_pump,
9117        stderr_pump,
9118        stderr_ring: Arc::clone(ring),
9119        spawned_at,
9120        spawned_at_ms,
9121        spawned_from,
9122        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9123        spawned_file_identity,
9124        process_start_time,
9125        process_identity,
9126        pid,
9127        roster_guard: Some(roster_guard),
9128        #[cfg(target_os = "macos")]
9129        privacy_exec,
9130        #[cfg(target_os = "macos")]
9131        report_ready: Arc::new(OnceLock::new()),
9132        spawn_failure: None,
9133    })
9134}
9135
9136#[cfg(test)]
9137mod spawn_record_tests {
9138    use super::*;
9139
9140    thread_local! {
9141        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9142    }
9143
9144    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9145        LAST_PID.set(child.id());
9146        #[cfg(windows)]
9147        if let Some((_, address)) = _spec
9148            .env
9149            .iter()
9150            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9151        {
9152            use std::io::{Read, Write};
9153            let mut stream = std::net::TcpStream::connect(address).unwrap();
9154            stream
9155                .write_all(&child.id().unwrap().to_ne_bytes())
9156                .unwrap();
9157            let mut release = [0];
9158            stream.read_exact(&mut release).unwrap();
9159        }
9160    }
9161
9162    #[cfg(windows)]
9163    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9164        if spec
9165            .env
9166            .iter()
9167            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9168        {
9169            Err(io::Error::other("injected module job assignment failure"))
9170        } else {
9171            Ok(())
9172        }
9173    }
9174
9175    fn spec() -> ModuleSpec {
9176        ModuleSpec {
9177            module_id: "spawn-record-worker".into(),
9178            #[cfg(unix)]
9179            program: "/bin/sleep".into(),
9180            #[cfg(windows)]
9181            program: std::env::current_exe().unwrap(),
9182            #[cfg(unix)]
9183            args: vec!["60".into()],
9184            #[cfg(windows)]
9185            args: vec![
9186                "--exact".into(),
9187                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9188                "--nocapture".into(),
9189            ],
9190            env: vec![],
9191            reserved: false,
9192            reserved_prefixes: vec![],
9193            protocol: ModuleProtocol::None,
9194            overlap: Default::default(),
9195        }
9196    }
9197
9198    #[tokio::test]
9199    async fn record_sync_failure_fails_spawn_and_reaps_child() {
9200        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9201        let supervisor =
9202            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9203                .with_live_children_record(home.join("live-children.json"));
9204        crate::live_children::fail_next_file_sync();
9205        let result = supervisor.spawn(spec());
9206        let failed = result.is_err();
9207        if let Ok(module) = result {
9208            module.stop().await.unwrap();
9209        }
9210        assert!(failed, "record failure must fail spawn");
9211        let pid = LAST_PID.get().expect("the real OS child was created");
9212        assert!(
9213            subc_os::Process::open(pid).unwrap().is_none(),
9214            "child {pid} survived rejected admission or was not reaped"
9215        );
9216    }
9217
9218    #[cfg(target_os = "macos")]
9219    #[test]
9220    #[ignore = "manual on-device latency measurement"]
9221    fn durable_record_write_latency_50() {
9222        let home = cortexkit_test_support::ScratchDir::new("record-latency");
9223        let path = home.join("live-children.json");
9224        let children = vec![crate::live_children::LiveChild {
9225            module_id: "latency-worker".into(),
9226            pid: std::process::id(),
9227            protocol: ModuleProtocol::None,
9228            start_time: Some(1),
9229            executable: None,
9230            cgroup_name: None,
9231        }];
9232        let mut samples = Vec::new();
9233        for _ in 0..50 {
9234            let start = std::time::Instant::now();
9235            crate::live_children::write_record(&path, &children).unwrap();
9236            samples.push(start.elapsed());
9237        }
9238        samples.sort();
9239        eprintln!(
9240            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9241            (samples[24] + samples[25]) / 2,
9242            samples[47]
9243        );
9244    }
9245
9246    #[cfg(windows)]
9247    #[test]
9248    fn windows_store_touch_fixture() {
9249        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9250            std::fs::write(path, b"module ran").unwrap();
9251        }
9252    }
9253
9254    #[cfg(windows)]
9255    #[tokio::test]
9256    async fn windows_daemon_fixture() {
9257        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9258            return;
9259        };
9260        let mut spec = spec();
9261        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9262        let supervisor =
9263            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9264        let _module = supervisor.spawn(spec).unwrap();
9265    }
9266
9267    #[cfg(windows)]
9268    #[test]
9269    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9270        use std::io::Read;
9271        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9272        // An outer job models a scheduler/service host. The daemon's root and
9273        // root job must nest without requiring breakaway from this job.
9274        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9275        command
9276            .args([
9277                "--exact",
9278                "supervise::spawn_record_tests::windows_daemon_fixture",
9279                "--nocapture",
9280            ])
9281            .env(
9282                "SUBC_TEST_DAEMON_BARRIER",
9283                listener.local_addr().unwrap().to_string(),
9284            );
9285        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9286        listener.set_nonblocking(true).unwrap();
9287        let deadline = std::time::Instant::now() + Duration::from_secs(30);
9288        let mut stream = loop {
9289            if let Ok((stream, _)) = listener.accept() {
9290                break stream;
9291            }
9292            assert!(
9293                daemon.child.try_wait().unwrap().is_none(),
9294                "daemon failed before rendezvous"
9295            );
9296            assert!(
9297                std::time::Instant::now() < deadline,
9298                "daemon did not reach CreateProcess failpoint"
9299            );
9300            std::thread::yield_now();
9301        };
9302        stream
9303            .set_read_timeout(Some(Duration::from_secs(30)))
9304            .unwrap();
9305        let mut pid = [0; 4];
9306        stream.read_exact(&mut pid).unwrap();
9307        let pid = u32::from_ne_bytes(pid);
9308        assert!(subc_jobobject::process_exists(pid));
9309        // Terminate only the daemon, not its outer job: otherwise the host job
9310        // would also kill a child whose daemon never joined its root job.
9311        daemon.child.kill().unwrap();
9312        daemon.child.wait().unwrap();
9313        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9314        if !exited {
9315            daemon.job.terminate().unwrap();
9316        }
9317        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9318    }
9319
9320    #[cfg(windows)]
9321    #[tokio::test]
9322    async fn windows_containment_failure_never_resumes() {
9323        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9324        let sentinel = home.join("store-touched");
9325        let mut spec = spec();
9326        spec.env
9327            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9328        spec.env.push((
9329            "SUBC_TEST_STORE_TOUCH".into(),
9330            sentinel.display().to_string(),
9331        ));
9332        let supervisor =
9333            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9334        let result = supervisor.spawn(spec);
9335        // A version that resumes despite assignment failure must have time to
9336        // execute its sentinel before dropping the returned module kills it.
9337        if let Ok(module) = result {
9338            let _ = tokio::time::timeout(Duration::from_secs(10), async {
9339                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9340                    tokio::task::yield_now().await;
9341                }
9342            })
9343            .await;
9344            let _ = module.stop().await;
9345            panic!(
9346                "assignment failure resumed a module; store touched={}",
9347                sentinel.exists()
9348            );
9349        }
9350        assert!(
9351            !sentinel.exists(),
9352            "refused module executed store-touch sentinel"
9353        );
9354        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9355    }
9356
9357    #[cfg(windows)]
9358    #[tokio::test]
9359    async fn windows_nested_daemon_fixture() {
9360        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9361            return;
9362        };
9363        let mut spec = spec();
9364        spec.env.push((
9365            "SUBC_TEST_STORE_TOUCH".into(),
9366            sentinel.to_str().unwrap().into(),
9367        ));
9368        let supervisor =
9369            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9370        let module = supervisor
9371            .spawn(spec)
9372            .expect("per-module job must nest below daemon and host jobs");
9373        tokio::time::timeout(Duration::from_secs(10), async {
9374            while !PathBuf::from(&sentinel).exists() {
9375                tokio::task::yield_now().await;
9376            }
9377        })
9378        .await
9379        .unwrap();
9380        module.stop().await.unwrap();
9381    }
9382
9383    #[cfg(windows)]
9384    #[test]
9385    fn windows_module_job_nests_under_host_job() {
9386        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9387        let sentinel = home.join("nested-store-touched");
9388        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9389        command
9390            .args([
9391                "--exact",
9392                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9393                "--nocapture",
9394            ])
9395            .env("SUBC_TEST_NESTED_STORE", &sentinel);
9396        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9397        assert!(daemon.child.wait().unwrap().success());
9398        assert!(sentinel.exists(), "nested protocol:none module never ran");
9399    }
9400}
9401
9402#[cfg(target_os = "linux")]
9403pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9404    use subc_cgroup::KillOutcome;
9405    match subc_cgroup::kill_module(placement, module_id) {
9406        KillOutcome::Killed => {}
9407        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9408            debug!(
9409                module_id,
9410                "cgroup tree kill unavailable; using direct-child kill"
9411            );
9412        }
9413        KillOutcome::IoError { path, error } => {
9414            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9415        }
9416    }
9417}
9418
9419/// Assign a freshly spawned Windows child without ever resuming on failure.
9420///
9421/// Steps 2 and 3 of the suspended-create contract: the job is created and the
9422/// child assigned **while it is still suspended** (step 1 is
9423/// `suspend_on_create_async` at the spawn site). The caller resumes only after
9424/// the child's roster admission is durable. Failed containment kills and reaps
9425/// the suspended child instead of running an unprotected module.
9426#[cfg(windows)]
9427fn contain_spawned_child(
9428    child: &mut Child,
9429    spec: &ModuleSpec,
9430) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9431    let assignment = || {
9432        let job = subc_jobobject::JobObject::new()?;
9433        #[cfg(test)]
9434        spawn_record_tests::assignment_failpoint(spec)?;
9435        job.assign(child)?;
9436        Ok::<_, io::Error>(job)
9437    };
9438    match assignment() {
9439        Ok(job) => Ok(Some(job)),
9440        Err(source) => {
9441            kill_and_reap_rejected_child(child);
9442            Err(SuperviseError::Spawn {
9443                program: spec.program.clone(),
9444                source: io::Error::other(format!(
9445                    "Windows module job containment failed: {source}"
9446                )),
9447                cgroup_path: None,
9448            })
9449        }
9450    }
9451}
9452
9453/// Rejection is synchronous: do not return a spawn error while the child can
9454/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
9455fn kill_and_reap_rejected_child(child: &mut Child) {
9456    #[cfg(unix)]
9457    if let Some(pid) = child
9458        .id()
9459        .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9460    {
9461        let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9462    }
9463    let _ = child.start_kill();
9464    while matches!(child.try_wait(), Ok(None)) {
9465        std::thread::yield_now();
9466    }
9467}
9468
9469/// Resume a suspended child, killing it if it cannot be started.
9470///
9471/// A suspended process holds a pid and does nothing, so there is no useful
9472/// state to return: the caller gets an error and the spawn fails.
9473#[cfg(windows)]
9474fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9475    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9476        // Kill it here rather than leaving a suspended process for the caller
9477        // to notice; `kill_on_drop` would eventually do this, but the module
9478        // would have been reported as running in between.
9479        let _ = std::process::Command::new("taskkill.exe")
9480            .args(["/PID", &pid.to_string(), "/T", "/F"])
9481            .stdin(Stdio::null())
9482            .stdout(Stdio::null())
9483            .stderr(Stdio::null())
9484            .status();
9485        return Err(SuperviseError::Spawn {
9486            program: spec.program.clone(),
9487            source,
9488            cgroup_path: None,
9489        });
9490    }
9491    Ok(())
9492}
9493
9494#[cfg(target_os = "linux")]
9495fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9496    match placement.remove_module(module_id) {
9497        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9498        Err(error) => warn!(
9499            module_id,
9500            error = %error,
9501            "could not remove module cgroup after process exit; continuing teardown"
9502        ),
9503    }
9504}
9505
9506#[cfg(target_os = "linux")]
9507async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9508    // Reaping the direct child is not proof its descendants exited. End the
9509    // residual tree and wait for the kernel's population fact before rmdir;
9510    // otherwise a successful parent wait leaks a directory on each restart.
9511    if matches!(
9512        subc_cgroup::kill_module(Some(placement), module_id),
9513        subc_cgroup::KillOutcome::Killed
9514    ) {
9515        if let Ok(path) = placement.module_path(module_id) {
9516            while std::fs::read_to_string(path.join("cgroup.events"))
9517                .ok()
9518                .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9519            {
9520                sleep(Duration::from_millis(1)).await;
9521            }
9522        }
9523    }
9524    remove_module_cgroup(placement, module_id);
9525}
9526
9527#[cfg(target_os = "linux")]
9528fn apply_cgroup_placement(
9529    command: &mut Command,
9530    spec: &ModuleSpec,
9531    path: &std::path::Path,
9532) -> Result<(), SuperviseError> {
9533    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9534        module_id: spec.module_id.clone(),
9535        source,
9536    })
9537}
9538
9539fn capture_retention(spec: &ModuleSpec) -> Retention {
9540    let defaults = Retention::default();
9541    let value = |name: &str| {
9542        spec.env
9543            .iter()
9544            .rev()
9545            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9546    };
9547    Retention {
9548        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9549            .and_then(|value| value.parse().ok())
9550            .unwrap_or(defaults.max_file_mb),
9551        keep: value(CAPTURE_KEEP_ENV)
9552            .and_then(|value| value.parse().ok())
9553            .unwrap_or(defaults.keep),
9554        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9555            .and_then(|value| value.parse().ok())
9556            .unwrap_or(defaults.max_age_days),
9557    }
9558}
9559
9560/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
9561/// module's registration to the exact process the supervisor spawned.
9562fn generate_launch_nonce() -> Result<String, SuperviseError> {
9563    let mut bytes = [0u8; 32];
9564    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9565        reason: source.to_string(),
9566    })?;
9567    let mut hex = String::with_capacity(64);
9568    for b in bytes {
9569        use std::fmt::Write;
9570        let _ = write!(hex, "{b:02x}");
9571    }
9572    Ok(hex)
9573}
9574
9575/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
9576/// signal about how many leading bytes matched.
9577fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9578    if a.len() != b.len() {
9579        return false;
9580    }
9581    let mut diff = 0u8;
9582    for (x, y) in a.iter().zip(b.iter()) {
9583        diff |= x ^ y;
9584    }
9585    diff == 0
9586}
9587
9588/// The kernel's image after an acknowledged exec, shared by ordinary launches
9589/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
9590/// not identities inferred from a configured pathname.
9591#[cfg(not(windows))]
9592fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9593    let process = subc_os::Process::open(pid).ok().flatten()?;
9594    #[cfg(target_os = "linux")]
9595    return sample_after_linux_exec(
9596        || {
9597            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9598            let committed = linux_exec_committed(&stat);
9599            #[cfg(test)]
9600            spawn_image_tests::report_exec_state(committed);
9601            committed
9602        },
9603        || process.observe(),
9604    );
9605    #[cfg(not(target_os = "linux"))]
9606    process.observe()
9607}
9608
9609/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
9610/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
9611/// installation. Until it clears, /proc/pid/exe can still name the daemon.
9612#[cfg(target_os = "linux")]
9613fn linux_exec_committed(stat: &str) -> Option<bool> {
9614    const PF_FORKNOEXEC: u64 = 0x40;
9615    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9616    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9617}
9618
9619#[cfg(target_os = "linux")]
9620fn sample_after_linux_exec(
9621    mut exec_committed: impl FnMut() -> Option<bool>,
9622    observe: impl FnOnce() -> Option<subc_os::Observation>,
9623) -> Option<subc_os::Observation> {
9624    let deadline = std::time::Instant::now() + Duration::from_secs(1);
9625    while !exec_committed()? {
9626        // A failed or indefinitely stalled exec must not publish the inherited
9627        // image as authority to signal a future orphan. Missing identity keeps
9628        // the existing conservative orphan-sweep behavior.
9629        if std::time::Instant::now() >= deadline {
9630            return None;
9631        }
9632        std::thread::sleep(Duration::from_micros(50));
9633    }
9634    observe()
9635}
9636
9637#[cfg(all(test, target_os = "linux"))]
9638mod spawn_image_tests {
9639    use super::*;
9640    use std::cell::{Cell, RefCell};
9641
9642    enum ExecEvent {
9643        Uncommitted,
9644        Image(Option<subc_os::Observation>),
9645    }
9646
9647    thread_local! {
9648        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9649    }
9650
9651    pub(super) fn report_exec_state(committed: Option<bool>) {
9652        if committed == Some(false) {
9653            EXEC_EVENTS.with(|events| {
9654                if let Some(events) = events.borrow_mut().take() {
9655                    let _ = events.send(ExecEvent::Uncommitted);
9656                }
9657            });
9658        }
9659    }
9660
9661    #[test]
9662    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9663        use std::io::{Read, Write};
9664        use std::os::fd::AsRawFd;
9665
9666        let descriptor = std::fs::File::open("/dev/null").unwrap();
9667        let mut command = std::process::Command::new("/bin/sleep");
9668        command.arg("30");
9669        let (mut ready, mut release) =
9670            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9671                .unwrap();
9672        let spawn = std::thread::spawn(move || command.spawn());
9673        let mut message = [0u8; 8];
9674        ready.read_exact(&mut message).unwrap();
9675        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9676        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9677        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9678        let (events, received) = std::sync::mpsc::channel();
9679        let observer = std::thread::spawn(move || {
9680            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9681            let image = observe_spawned_image(pid);
9682            let _ = events.send(ExecEvent::Image(image));
9683        });
9684        // Release only once the production observer has seen the uncommitted
9685        // exec, or has incorrectly returned the inherited image. Either way,
9686        // release and reap before asserting so a regression cannot leak a child.
9687        let first = received.recv_timeout(Duration::from_secs(5));
9688        release.write_all(&[1]).unwrap();
9689        let mut child = spawn.join().unwrap().unwrap();
9690        let after = match first {
9691            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9692                Ok(ExecEvent::Image(image)) => image,
9693                _ => None,
9694            },
9695            Ok(ExecEvent::Image(image)) => image,
9696            Err(_) => None,
9697        };
9698        observer.join().unwrap();
9699        child.kill().unwrap();
9700        child.wait().unwrap();
9701
9702        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9703        assert_eq!(
9704            inherited.unwrap().executable,
9705            subc_os::file_identity(&std::env::current_exe().unwrap())
9706        );
9707        assert_eq!(
9708            after.unwrap().executable,
9709            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9710        );
9711    }
9712
9713    #[test]
9714    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9715        let committed = Cell::new(false);
9716        let inherited = subc_os::Observation {
9717            start_time: 23,
9718            executable: Some(subc_os::FileIdentity {
9719                device: 29,
9720                inode: 31,
9721            }),
9722        };
9723        let module = subc_os::Observation {
9724            start_time: 23,
9725            executable: Some(subc_os::FileIdentity {
9726                device: 29,
9727                inode: 37,
9728            }),
9729        };
9730        let polls = Cell::new(0);
9731        let image = sample_after_linux_exec(
9732            || {
9733                polls.set(polls.get() + 1);
9734                // Hold the inherited image through the first poll, as in the
9735                // posix_spawn parent-wake window, then commit the module image.
9736                committed.set(polls.get() == 2);
9737                Some(committed.get())
9738            },
9739            || Some(if committed.get() { module } else { inherited }),
9740        );
9741        assert_eq!(image, Some(module));
9742        assert_eq!(polls.get(), 2);
9743    }
9744
9745    #[test]
9746    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9747        assert_eq!(
9748            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9749            Some(false)
9750        );
9751        assert_eq!(
9752            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9753            Some(true)
9754        );
9755        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9756        assert_eq!(linux_exec_committed("17 (a) R"), None);
9757    }
9758
9759    #[test]
9760    fn unreadable_exec_state_never_samples_an_inherited_image() {
9761        assert_eq!(
9762            sample_after_linux_exec(
9763                || None,
9764                || panic!("must not sample before exec confirmation")
9765            ),
9766            None
9767        );
9768    }
9769
9770    #[test]
9771    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9772        assert_eq!(
9773            sample_after_linux_exec(
9774                || Some(false),
9775                || panic!("must not sample after an exec timeout")
9776            ),
9777            None
9778        );
9779    }
9780}
9781
9782/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
9783/// different executable after the no-delete capture was taken.
9784#[cfg(windows)]
9785fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9786    let mut program = spec.program.clone();
9787    if program.is_absolute() || program.components().count() > 1 {
9788        // An explicit path can name a PE image with no extension. Try that
9789        // exact file first, as Command does, before falling back to .exe.
9790        match subc_os::ExecutableCapture::open(&program) {
9791            Ok(capture) => return Ok(capture),
9792            Err(error)
9793                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9794            Err(error) => return Err(error),
9795        }
9796        program.set_extension("exe");
9797        return subc_os::ExecutableCapture::open(&program);
9798    }
9799    if program.extension().is_none() {
9800        program.set_extension("exe");
9801    }
9802    let custom_path = spec
9803        .env
9804        .iter()
9805        .rev()
9806        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9807    let mut directories = Vec::new();
9808    // These are Windows' default executable search locations. A configured
9809    // PATH still overrides the inherited PATH, not the system directories.
9810    if let Some(parent) = std::env::current_exe()?.parent() {
9811        directories.push(parent.to_path_buf());
9812    }
9813    directories.push(std::env::current_dir()?);
9814    if let Some(windows) = std::env::var_os("SystemRoot") {
9815        let windows = PathBuf::from(windows);
9816        directories.push(windows.join("System32"));
9817        directories.push(windows.clone());
9818    }
9819    let path = custom_path
9820        .map(|(_, value)| std::ffi::OsString::from(value))
9821        .or_else(|| std::env::var_os("PATH"));
9822    if let Some(path) = path {
9823        directories.extend(std::env::split_paths(&path));
9824    }
9825    for directory in directories {
9826        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9827            Ok(capture) => return Ok(capture),
9828            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9829            Err(error) => return Err(error),
9830        }
9831    }
9832    Err(io::Error::new(
9833        io::ErrorKind::NotFound,
9834        "resolved executable not found",
9835    ))
9836}
9837
9838fn spawn_and_mark_running(
9839    spec: &ModuleSpec,
9840    runtime: &SupervisorRuntimeConfig,
9841    snapshot: &SharedSnapshot,
9842) -> Result<SupervisedChild, SuperviseError> {
9843    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9844    let child = spawn_child(
9845        spec,
9846        runtime.connection_file_path.as_deref(),
9847        runtime.supervisor_handle.as_ref(),
9848        &runtime.stderr_ring,
9849        runtime.capture_logs_dir.as_deref(),
9850        &runtime.child_roster,
9851        #[cfg(target_os = "linux")]
9852        runtime.cgroup_placement.as_ref(),
9853    )?;
9854    set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9855    if let Some(spawn) = activity_spawn {
9856        spawn.complete(snapshot);
9857    }
9858    Ok(child)
9859}
9860
9861enum RegistrationWaitOutcome {
9862    Registered,
9863    Exited(ExitReport),
9864    TimedOut,
9865}
9866
9867struct ReloadRegistrationFailure {
9868    exit_report: ExitReport,
9869    reason: String,
9870}
9871
9872#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9873enum BusyGaugeObservation {
9874    Quiescent,
9875    Busy,
9876    Omitted,
9877}
9878
9879/// What one health answer said about one declared busy gauge.
9880#[derive(Debug, Clone, PartialEq, Eq)]
9881enum GaugeVerdict {
9882    /// The gauge read 0: this gauge is not what holds the drain.
9883    Zero,
9884    /// The gauge read a positive count.
9885    NonZero(u64),
9886    /// The gauge was present but not a non-negative integer (`0.0`, a string,
9887    /// a negative number). It counts as busy, because the daemon cannot tell
9888    /// it is idle. Carries the value as JSON text, cut to a short prefix.
9889    NotACount(String),
9890    /// The health answer had no metrics object, or one without this gauge.
9891    Missing,
9892    /// The probe produced no health answer to read; carries the probe error's
9893    /// evidence label (`no-answer` for a probe that timed out).
9894    ProbeFailed(&'static str),
9895}
9896
9897impl GaugeVerdict {
9898    fn reads_quiet(&self) -> bool {
9899        matches!(self, Self::Zero)
9900    }
9901}
9902
9903impl fmt::Display for GaugeVerdict {
9904    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9905        match self {
9906            Self::Zero => f.write_str("zero"),
9907            Self::NonZero(value) => write!(f, "non_zero({value})"),
9908            Self::NotACount(value) => write!(f, "not_a_count({value})"),
9909            Self::Missing => f.write_str("missing_from_health_body"),
9910            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
9911        }
9912    }
9913}
9914
9915/// The longest JSON text kept for a gauge value that is not a count, so a
9916/// module reporting a large object as a gauge cannot flood the drain log.
9917const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
9918
9919/// Read each declared gauge out of one health answer's metrics.
9920fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
9921    let metrics = metrics.and_then(Value::as_object);
9922    gauges
9923        .iter()
9924        .map(
9925            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
9926                None => GaugeVerdict::Missing,
9927                Some(value) => match value.as_u64() {
9928                    Some(0) => GaugeVerdict::Zero,
9929                    Some(count) => GaugeVerdict::NonZero(count),
9930                    None => {
9931                        let mut text = value.to_string();
9932                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
9933                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
9934                            while !text.is_char_boundary(end) {
9935                                end -= 1;
9936                            }
9937                            text.truncate(end);
9938                            text.push_str("...");
9939                        }
9940                        GaugeVerdict::NotACount(text)
9941                    }
9942                },
9943            },
9944        )
9945        .collect()
9946}
9947
9948/// The drain's overall reading of one health answer: omitted when any
9949/// declared gauge is missing, quiescent only when every gauge reads 0.
9950fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
9951    if verdicts
9952        .iter()
9953        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
9954    {
9955        BusyGaugeObservation::Omitted
9956    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
9957        BusyGaugeObservation::Quiescent
9958    } else {
9959        BusyGaugeObservation::Busy
9960    }
9961}
9962
9963/// One declared gauge's history over a drain: the latest verdict on it, and
9964/// the latest value actually read for it (which a later failed probe or
9965/// missing gauge does not erase). Times are offsets from the start of the
9966/// drain's wait.
9967#[derive(Debug, Clone, Default, PartialEq, Eq)]
9968struct GaugeTrace {
9969    latest: Option<(GaugeVerdict, Duration)>,
9970    last_value: Option<(GaugeVerdict, Duration)>,
9971}
9972
9973/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
9974/// its ceiling can say in its timeout line which declared gauge held it and
9975/// why. Without it, a gauge that read busy, a gauge the module never
9976/// reported, and a probe that never got an answer all looked the same: a
9977/// drain that ended at its ceiling with nothing in flight.
9978#[derive(Debug, Clone, Default, PartialEq, Eq)]
9979struct DrainGaugeTrace {
9980    /// Health probes the drain sent.
9981    probes: u32,
9982    /// Each declared gauge, in declaration order.
9983    gauges: Vec<(String, GaugeTrace)>,
9984}
9985
9986impl DrainGaugeTrace {
9987    fn new(gauges: &[String]) -> Self {
9988        Self {
9989            probes: 0,
9990            gauges: gauges
9991                .iter()
9992                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
9993                .collect(),
9994        }
9995    }
9996
9997    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
9998    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
9999        self.probes = self.probes.saturating_add(1);
10000        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10001            trace.latest = Some((verdict.clone(), at));
10002            if matches!(
10003                verdict,
10004                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10005            ) {
10006                trace.last_value = Some((verdict.clone(), at));
10007            }
10008        }
10009    }
10010}
10011
10012/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
10013/// declared gauge, separated by ` | `. A value is shown as its verdict
10014/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
10015/// and whether that counted as busy.
10016impl fmt::Display for DrainGaugeTrace {
10017    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10018        if self.gauges.is_empty() {
10019            return f.write_str("none declared");
10020        }
10021        for (index, (name, trace)) in self.gauges.iter().enumerate() {
10022            if index > 0 {
10023                f.write_str(" | ")?;
10024            }
10025            write!(f, "{name}: ")?;
10026            match &trace.latest {
10027                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10028                None => f.write_str("never probed")?,
10029            }
10030            match &trace.last_value {
10031                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10032                None => f.write_str("; no value ever read")?,
10033            }
10034        }
10035        Ok(())
10036    }
10037}
10038
10039pub(crate) fn declared_busy_gauges(
10040    registry: &Registry,
10041    module_id: &str,
10042) -> Result<Vec<String>, SuperviseError> {
10043    busy_gauges_of(
10044        registry
10045            .get_module(module_id)
10046            .map_err(SuperviseError::Registry)?,
10047    )
10048}
10049
10050/// [`declared_busy_gauges`] for the registration a connection holds, in any
10051/// slot: after cutover the incumbent is no longer the id's active
10052/// registration, and its own manifest is the one that names its gauges.
10053fn declared_busy_gauges_for_connection(
10054    registry: &Registry,
10055    connection_id: ConnectionId,
10056) -> Result<Vec<String>, SuperviseError> {
10057    busy_gauges_of(
10058        registry
10059            .get_module_by_connection(connection_id)
10060            .map_err(SuperviseError::Registry)?,
10061    )
10062}
10063
10064fn busy_gauges_of(
10065    registration: Option<crate::registry::ModuleRegistration>,
10066) -> Result<Vec<String>, SuperviseError> {
10067    let Some(registration) = registration else {
10068        return Ok(Vec::new());
10069    };
10070    let Some(self_signals) = registration.manifest.self_signals else {
10071        return Ok(Vec::new());
10072    };
10073
10074    let mut gauges = Vec::new();
10075    for declaration in self_signals {
10076        if declaration.kind != SelfSignalKind::Busy {
10077            continue;
10078        }
10079        match declaration.anchored_to {
10080            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10081                gauges.extend(declared)
10082            }
10083            _ => {
10084                // An invalid Busy anchor is fail-safe: the empty name cannot be
10085                // present in a conforming health report, so this drain stays busy.
10086                gauges.push(String::new());
10087            }
10088        }
10089    }
10090    Ok(gauges)
10091}
10092
10093/// Wait for `endpoint` to have nothing in flight and, when the module declares
10094/// busy gauges, for a health probe to report them quiet. The probe is addressed
10095/// by `scope`: a swap's superseded incumbent must be asked about its own
10096/// gauges, and by module id the probe would reach the promoted candidate.
10097/// While the gauges read busy, the probe repeats every
10098/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
10099/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
10100/// timeout line.
10101#[allow(clippy::too_many_arguments)]
10102async fn wait_for_forwarding_quiescence(
10103    forwarding: &ForwardingTable,
10104    module_id: &str,
10105    runtime: &SupervisorRuntimeConfig,
10106    endpoint: crate::ModuleEndpointId,
10107    deadline: Instant,
10108    busy_gauges: &[String],
10109    scope: DrainScope,
10110    trace: &mut DrainGaugeTrace,
10111) -> Result<bool, SuperviseError> {
10112    let wait_started = Instant::now();
10113    let mut gauges_quiescent = busy_gauges.is_empty();
10114    let mut next_probe_at = wait_started;
10115    let mut omission_reported = false;
10116
10117    loop {
10118        let now = Instant::now();
10119        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10120            let report = match scope {
10121                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10122                DrainScope::Endpoint(endpoint) => {
10123                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10124                }
10125            };
10126            let verdicts = match &report {
10127                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10128                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10129            };
10130            trace.record(&verdicts, wait_started.elapsed());
10131            gauges_quiescent = match report {
10132                Ok(_) => match busy_gauge_observation(&verdicts) {
10133                    BusyGaugeObservation::Quiescent => true,
10134                    BusyGaugeObservation::Busy => false,
10135                    BusyGaugeObservation::Omitted => {
10136                        // Once per drain: a module that never reports a gauge
10137                        // it declared would otherwise say so on every probe.
10138                        if !omission_reported {
10139                            forwarding
10140                                .counters()
10141                                .increment_drains_with_undeclared_gauge();
10142                            let missing = busy_gauges
10143                                .iter()
10144                                .zip(&verdicts)
10145                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10146                                .map(|(gauge, _)| gauge.as_str())
10147                                .collect::<Vec<_>>()
10148                                .join(",");
10149                            warn!(
10150                                module_id,
10151                                missing_gauges = %missing,
10152                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10153                            );
10154                            omission_reported = true;
10155                        }
10156                        false
10157                    }
10158                },
10159                Err(err) => {
10160                    warn!(
10161                        module_id,
10162                        error = %err,
10163                        "drain health.check did not produce declared busy gauges; treating module as busy"
10164                    );
10165                    false
10166                }
10167            };
10168            // Counted from the probe's completion: the probe above was awaited,
10169            // so no second probe can start while one is still outstanding.
10170            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10171        }
10172
10173        let in_flight = forwarding
10174            .endpoint_in_flight_count(endpoint)
10175            .map_err(SuperviseError::Forwarding)?;
10176        if in_flight == 0 && gauges_quiescent {
10177            return Ok(true);
10178        }
10179
10180        let now = Instant::now();
10181        if now >= deadline {
10182            return Ok(false);
10183        }
10184        let mut wait = deadline
10185            .saturating_duration_since(now)
10186            .min(REGISTRY_RELEASE_POLL);
10187        if !busy_gauges.is_empty() {
10188            wait = wait.min(next_probe_at.saturating_duration_since(now));
10189        }
10190        sleep(wait).await;
10191    }
10192}
10193
10194/// The wait between drain busy-gauge probes: the larger of
10195/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
10196/// drain's own in-flight poll interval.
10197fn drain_busy_reprobe_interval() -> Duration {
10198    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10199}
10200
10201/// Logs why a drain gave up: the drain waited its full timeout without the
10202/// module going quiet, and teardown is being forced.
10203///
10204/// The line names two kinds of evidence, because a bare "timed out" hides
10205/// causes that need different fixes:
10206/// - `holdouts`: the requests and routes still open when the timeout hit,
10207///   and on which connections. One stuck call is a module bug; a held stream
10208///   on every session is a module that should end its streams when it
10209///   receives `module.draining`.
10210/// - `gauges`: for each busy gauge the module declares, its latest reading
10211///   and how many probes ran. This separates a module that kept reporting
10212///   itself busy, one whose health answer never contained the gauge, and one
10213///   that never answered the probes, which otherwise all look like a timeout
10214///   with nothing in flight.
10215fn warn_drain_timed_out(
10216    module_id: &str,
10217    drain_timeout: Duration,
10218    reason: RouteCloseReason,
10219    holdouts: &crate::forwarding::DrainHoldouts,
10220    gauges: &DrainGaugeTrace,
10221) {
10222    warn!(
10223        module_id = %module_id,
10224        waited = ?drain_timeout,
10225        ?reason,
10226        held_requests = holdouts.requests,
10227        held_routes = holdouts.routes,
10228        total_routes = holdouts.total_routes,
10229        top_connections = ?holdouts.top_connections,
10230        // `module_channel:corr`, so the module can find each held request
10231        // in its own log; capped, so `held_requests` is the full count.
10232        held = %holdouts
10233            .held
10234            .iter()
10235            .map(|(channel, corr)| format!("{channel}:{corr}"))
10236            .collect::<Vec<_>>()
10237            .join(","),
10238        gauge_probes = gauges.probes,
10239        busy_gauges = %gauges,
10240        "route drain timed out before request quiescence; forcing teardown"
10241    );
10242}
10243
10244/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
10245///
10246/// `Ok` is always honest and passed straight through -- the wait actually measured
10247/// in-flight state. `Err` means the wait produced no measurement at all (the
10248/// forwarding table's lock was poisoned), so `false` is reported as the one honest
10249/// constant: the drain did not complete. Never recomputed from route state, never a
10250/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
10251fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
10252    match wait_result {
10253        Ok(drained) => *drained,
10254        Err(_) => false,
10255    }
10256}
10257
10258fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
10259    for released in released_routes {
10260        let frame = match Frame::build_with_version(
10261            released.negotiated_ver,
10262            FrameType::Goodbye,
10263            control_flags(),
10264            released.channel,
10265            released.epoch,
10266            0,
10267            Vec::new(),
10268        ) {
10269            Ok(frame) => frame,
10270            Err(err) => {
10271                warn!(
10272                    route_channel = released.channel,
10273                    error = %err,
10274                    "failed to build supervisor drain route GOODBYE frame"
10275                );
10276                continue;
10277            }
10278        };
10279        if !released.close_on_delivery_failure() {
10280            crate::forwarding::send_module_route_goodbye(
10281                &forwarding.counters(),
10282                &released.sink,
10283                frame,
10284                released.module_id.as_deref(),
10285                "supervisor drain",
10286            );
10287            continue;
10288        }
10289        if let Err(err) = released.sink.try_send(frame) {
10290            warn!(
10291                target_connection_id = released.connection_id.get(),
10292                route_channel = released.channel,
10293                error = %err,
10294                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
10295            );
10296            let _ = forwarding.escalate_client_delivery_failure(
10297                released.connection_id,
10298                released.channel,
10299                released.epoch,
10300                CloseReason::new(
10301                    "route_goodbye_delivery_failed",
10302                    format!(
10303                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
10304                        released.channel
10305                    ),
10306                ),
10307                crate::forwarding::UndeliveredFrame {
10308                    module_id: released.module_id.as_deref(),
10309                    sink: &released.sink,
10310                },
10311            );
10312        }
10313    }
10314}
10315
10316fn send_module_draining(
10317    module_id: &str,
10318    reason: RouteCloseReason,
10319    deadline_ms: u64,
10320    target: &ModuleDrainTarget,
10321) {
10322    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
10323        reason,
10324        deadline_ms,
10325    }) {
10326        Ok(body) => body,
10327        Err(err) => {
10328            warn!(
10329                module_id,
10330                error = %err,
10331                "failed to encode module draining command"
10332            );
10333            return;
10334        }
10335    };
10336    let frame = match Frame::build_with_version(
10337        target.negotiated_ver,
10338        FrameType::Push,
10339        control_flags(),
10340        0,
10341        0,
10342        0,
10343        body,
10344    ) {
10345        Ok(frame) => frame,
10346        Err(err) => {
10347            warn!(
10348                module_id,
10349                error = %err,
10350                "failed to build module draining command frame"
10351            );
10352            return;
10353        }
10354    };
10355    if let Err(err) = target.sink.try_send(frame) {
10356        warn!(
10357            module_id,
10358            target_connection_id = target.endpoint.connection_id.get(),
10359            error = %err,
10360            "module draining command was not delivered to peer"
10361        );
10362    }
10363}
10364
10365/// The channel-0 GOODBYE that tells a module its stop is planned.
10366fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
10367    match Frame::build_with_version(
10368        negotiated_ver,
10369        FrameType::Goodbye,
10370        control_flags(),
10371        0,
10372        0,
10373        0,
10374        Vec::new(),
10375    ) {
10376        Ok(frame) => Some(frame),
10377        Err(err) => {
10378            warn!(
10379                module_id,
10380                error = %err,
10381                "failed to build module GOODBYE frame"
10382            );
10383            None
10384        }
10385    }
10386}
10387
10388/// Send every registered module connection its module GOODBYE at daemon
10389/// shutdown, then request that connection's close.
10390///
10391/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
10392/// before EOF, so the GOODBYE must reach the socket before the close. A close
10393/// request does not wait for the connection's queued frames: its writer gets a
10394/// bounded grace after the close, is aborted if it overruns it, and the daemon
10395/// process may exit before that grace ends. So with `wait_for_flush`, each
10396/// connection is closed only after its writer has acknowledged writing the
10397/// GOODBYE, or once a short shared budget runs out, so one module that is not
10398/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
10399/// are only queued, for a shutdown the operator has told to stop waiting.
10400/// A connection that is already gone is skipped.
10401#[cfg(unix)]
10402async fn send_module_goodbyes_for_daemon_shutdown(
10403    forwarding: &Arc<ForwardingTable>,
10404    reason: &CloseReason,
10405    wait_for_flush: bool,
10406) {
10407    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10408    let targets = match forwarding.module_connections() {
10409        Ok(targets) => targets,
10410        Err(err) => {
10411            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10412            return;
10413        }
10414    };
10415    let deadline = Instant::now() + GOODBYE_BUDGET;
10416    let mut sends = tokio::task::JoinSet::new();
10417    for target in targets {
10418        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10419            continue;
10420        };
10421        if !wait_for_flush {
10422            if let Err(err) = target.sink.try_send(frame) {
10423                debug!(
10424                    module_id = %target.module_id,
10425                    error = %err,
10426                    "shutdown module GOODBYE was not queued"
10427                );
10428            }
10429            continue;
10430        }
10431        let forwarding = Arc::clone(forwarding);
10432        let reason = reason.clone();
10433        sends.spawn(async move {
10434            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10435                Ok(Ok(())) => {}
10436                Ok(Err(err)) => debug!(
10437                    module_id = %target.module_id,
10438                    error = %err,
10439                    "module connection closed before its shutdown GOODBYE was written"
10440                ),
10441                Err(_) => warn!(
10442                    module_id = %target.module_id,
10443                    budget = ?GOODBYE_BUDGET,
10444                    "shutdown module GOODBYE was not written within its budget; closing anyway"
10445                ),
10446            }
10447            forwarding.request_connection_close(target.endpoint.connection_id, reason);
10448        });
10449    }
10450    // Every task ends by the shared deadline, so this wait is bounded too.
10451    while sends.join_next().await.is_some() {}
10452}
10453
10454fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10455    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10456        return;
10457    };
10458    if let Err(err) = target.sink.try_send(frame) {
10459        warn!(
10460            module_id,
10461            target_connection_id = target.endpoint.connection_id.get(),
10462            error = %err,
10463            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10464        );
10465        forwarding.request_connection_close(
10466            target.endpoint.connection_id,
10467            CloseReason::new(
10468                "module_goodbye_delivery_failed",
10469                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10470            ),
10471        );
10472    }
10473}
10474
10475#[derive(Clone, Copy)]
10476struct ForwardingDrainContext<'a> {
10477    spec: &'a ModuleSpec,
10478    runtime: &'a SupervisorRuntimeConfig,
10479    registry: &'a Registry,
10480    scope: DrainScope,
10481}
10482
10483/// Which process a forwarding drain addresses.
10484#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10485enum DrainScope {
10486    /// Whatever endpoint is active for the module id: every plain stop,
10487    /// restart and reload. Also moves the module's state to `Draining`.
10488    Active,
10489    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
10490    /// module id would resolve to the promoted candidate and leave neither
10491    /// process routable. The module's state is left alone, since the promoted
10492    /// candidate is what it describes and that process is running.
10493    Endpoint(crate::ModuleEndpointId),
10494}
10495
10496/// Whether a child being drained has already been asked to stop by the time
10497/// its drain wait starts.
10498///
10499/// The drain wait is the same budget whatever this says. What it decides is
10500/// whether the supervisor must ask by signal before that wait begins: a child
10501/// that nobody asked will sit out the whole budget and then be SIGKILLed,
10502/// healthy or not.
10503#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10504enum StopNotice {
10505    /// The module was sent `module.draining` and a module GOODBYE over its own
10506    /// registered connection, and stops itself.
10507    SentOverConnection,
10508    /// The forwarding drain found no registered connection for the module: a
10509    /// subc child spawned moments ago that has not sent HELLO yet, or a
10510    /// `protocol: "none"` child, which never registers.
10511    NoConnection,
10512    /// This path sends nothing over the module's connection: the supervisor has
10513    /// no forwarding table, or the caller stops the child without a forwarding
10514    /// drain.
10515    NotSent,
10516}
10517
10518async fn begin_forwarding_drain(
10519    spec: &ModuleSpec,
10520    runtime: &SupervisorRuntimeConfig,
10521    registry: &Registry,
10522    snapshot: &SharedSnapshot,
10523    enabled: Option<bool>,
10524    reason: RouteCloseReason,
10525) -> Result<StopNotice, SuperviseError> {
10526    let Some(forwarding) = runtime.forwarding.as_ref() else {
10527        return Err(SuperviseError::ReloadUnavailable {
10528            module_id: spec.module_id.clone(),
10529            reason: "supervisor was not configured with a forwarding table".to_string(),
10530        });
10531    };
10532
10533    begin_forwarding_drain_with(
10534        forwarding,
10535        ForwardingDrainContext {
10536            spec,
10537            runtime,
10538            registry,
10539            scope: DrainScope::Active,
10540        },
10541        snapshot,
10542        enabled,
10543        reason,
10544        runtime.drain_timeout,
10545    )
10546    .await
10547}
10548
10549async fn begin_forwarding_drain_if_configured(
10550    spec: &ModuleSpec,
10551    runtime: &SupervisorRuntimeConfig,
10552    registry: &Registry,
10553    snapshot: &SharedSnapshot,
10554    enabled: Option<bool>,
10555    reason: RouteCloseReason,
10556) -> Result<StopNotice, SuperviseError> {
10557    begin_forwarding_drain_with_timeout(
10558        spec,
10559        runtime,
10560        registry,
10561        snapshot,
10562        enabled,
10563        reason,
10564        runtime.drain_timeout,
10565    )
10566    .await
10567}
10568
10569/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
10570/// budget, for paths where the operator overrides the module's configured one
10571/// (`supervisor.restart{drain_timeout_ms}`).
10572async fn begin_forwarding_drain_with_timeout(
10573    spec: &ModuleSpec,
10574    runtime: &SupervisorRuntimeConfig,
10575    registry: &Registry,
10576    snapshot: &SharedSnapshot,
10577    enabled: Option<bool>,
10578    reason: RouteCloseReason,
10579    drain_timeout: Duration,
10580) -> Result<StopNotice, SuperviseError> {
10581    let Some(forwarding) = runtime.forwarding.as_ref() else {
10582        return Ok(StopNotice::NotSent);
10583    };
10584
10585    begin_forwarding_drain_with(
10586        forwarding,
10587        ForwardingDrainContext {
10588            spec,
10589            runtime,
10590            registry,
10591            scope: DrainScope::Active,
10592        },
10593        snapshot,
10594        enabled,
10595        reason,
10596        drain_timeout,
10597    )
10598    .await
10599}
10600
10601async fn begin_forwarding_drain_with(
10602    forwarding: &ForwardingTable,
10603    context: ForwardingDrainContext<'_>,
10604    snapshot: &SharedSnapshot,
10605    enabled: Option<bool>,
10606    reason: RouteCloseReason,
10607    drain_timeout: Duration,
10608) -> Result<StopNotice, SuperviseError> {
10609    let ForwardingDrainContext {
10610        spec,
10611        runtime,
10612        registry,
10613        scope,
10614    } = context;
10615    debug_assert_ne!(reason, RouteCloseReason::Crash);
10616    let terminal = matches!(reason, RouteCloseReason::Disable);
10617    let drain_started_at = Instant::now();
10618    let drain_deadline = drain_started_at + drain_timeout;
10619    let deadline_ms =
10620        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10621    let busy_gauges = match scope {
10622        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10623        DrainScope::Endpoint(endpoint) => {
10624            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10625        }
10626    };
10627
10628    // Admission gate first: route.open/commit and route REQUEST admission are closed
10629    // before the first quiescence check, so the outstanding count can only fall.
10630    let gate_started = Instant::now();
10631    let drain_target = match scope {
10632        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10633        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10634    }
10635    .map_err(SuperviseError::Forwarding)?;
10636    // The instant admission closed, and how long taking the forwarding write
10637    // lock to close it took. The timeout line reports only the quiescence
10638    // wait, so without this a drain that started late looked like one that
10639    // started on time.
10640    info!(
10641        module_id = %spec.module_id,
10642        ?reason,
10643        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10644        connected = drain_target.is_some(),
10645        "module drain began; route admission closed"
10646    );
10647    if scope == DrainScope::Active {
10648        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10649            state.state = ModuleState::Draining;
10650            state.draining_to_replace =
10651                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10652            if let Some(enabled) = enabled {
10653                state.enabled = enabled;
10654            }
10655        })?;
10656    }
10657
10658    let Some(target) = drain_target.as_ref() else {
10659        // Nothing was sent: the module has no registered connection to carry
10660        // `module.draining` or a GOODBYE. The caller must not assume the child
10661        // was asked to stop.
10662        return Ok(StopNotice::NoConnection);
10663    };
10664    {
10665        send_module_draining(&spec.module_id, reason, deadline_ms, target);
10666        #[cfg(feature = "test-support")]
10667        forwarding.wait_before_route_closing_for_test().await;
10668        let routes = forwarding
10669            .endpoint_routes(target.endpoint)
10670            .map_err(SuperviseError::Forwarding)?;
10671        let routes_notified = routes.len();
10672        crate::control::send_route_control_pushes(
10673            forwarding,
10674            routes.clone(),
10675            ClientControlPush::RouteClosing {
10676                module_id: spec.module_id.clone(),
10677                channels: Vec::new(),
10678                reason,
10679            },
10680        );
10681        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10682
10683        // `route.closing` was just sent above: from here on every return path,
10684        // including an early one, MUST send `route.closed` before propagating
10685        // anything else. A client holds `closing` as a promise that a verdict is
10686        // coming; leaving early without `closed` strands it waiting forever, since
10687        // `closing` carries no timeout of its own.
10688        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10689        let wait_result = wait_for_forwarding_quiescence(
10690            forwarding,
10691            &spec.module_id,
10692            runtime,
10693            target.endpoint,
10694            drain_deadline,
10695            &busy_gauges,
10696            scope,
10697            &mut gauge_trace,
10698        )
10699        .await;
10700        let drained = drained_after_quiescence_wait(&wait_result);
10701        if let Err(err) = &wait_result {
10702            error!(
10703                module_id = %spec.module_id,
10704                ?reason,
10705                error = %err,
10706                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10707            );
10708        } else if !drained {
10709            // Read before teardown releases the routes.
10710            let holdouts = forwarding
10711                .endpoint_drain_holdouts(target.endpoint)
10712                .unwrap_or_default();
10713            warn_drain_timed_out(
10714                &spec.module_id,
10715                drain_timeout,
10716                reason,
10717                &holdouts,
10718                &gauge_trace,
10719            );
10720        }
10721        crate::control::send_route_control_pushes(
10722            forwarding,
10723            routes,
10724            ClientControlPush::RouteClosed {
10725                module_id: spec.module_id.clone(),
10726                channels: Vec::new(),
10727                reason,
10728                drained,
10729                abandoned: target.abandoned_bindings.len() as u32,
10730                excluded_subscriptions: target.excluded_subscriptions,
10731                terminal: Some(terminal),
10732            },
10733        );
10734        wait_result?;
10735
10736        // `route.closed` has now been sent unconditionally above. From here the
10737        // remaining steps are cleanup (route + module GOODBYE) rather than a
10738        // promise the client is waiting on, but a lock-poisoned
10739        // `release_module_endpoint_routes` would otherwise skip the module
10740        // GOODBYE silently too -- send it before propagating the error.
10741        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10742            Ok(routes) => routes,
10743            Err(err) => {
10744                warn!(
10745                    module_id = %spec.module_id,
10746                    ?reason,
10747                    error = %err,
10748                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10749                );
10750                send_module_goodbye(&spec.module_id, forwarding, target);
10751                return Err(SuperviseError::Forwarding(err));
10752            }
10753        };
10754        let route_goodbye_count = released_routes.len();
10755        send_route_goodbyes(forwarding, released_routes);
10756        send_module_goodbye(&spec.module_id, forwarding, target);
10757
10758        // The drain's happy path was previously silent: every emission above is
10759        // best-effort with only its failure arm logged, so "were consumers told"
10760        // was unprovable from the daemon log (surfaced by a 30-minute consumer
10761        // hang where the open question was exactly whether teardown notice went
10762        // out). One summary line makes that class decidable in one grep.
10763        info!(
10764            module_id = %spec.module_id,
10765            ?reason,
10766            routes_notified,
10767            route_goodbyes = route_goodbye_count,
10768            abandoned_reservations = target.abandoned_bindings.len(),
10769            excluded_subscriptions = target.excluded_subscriptions,
10770            drained,
10771            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10772        );
10773    }
10774
10775    Ok(StopNotice::SentOverConnection)
10776}
10777
10778/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
10779/// the only slot a plain (non-swap) spawn can register into.
10780async fn wait_for_registration_after_reload(
10781    registry: &Registry,
10782    module_id: &str,
10783    snapshot: &SharedSnapshot,
10784    child: &mut SupervisedChild,
10785    wait: Duration,
10786) -> Result<RegistrationWaitOutcome, SuperviseError> {
10787    wait_for_slot_registration(
10788        registry,
10789        crate::registry::RegistrationSlot::Active(module_id),
10790        module_id,
10791        snapshot,
10792        child,
10793        wait,
10794    )
10795    .await
10796}
10797
10798/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
10799///
10800/// Keyed on the slot rather than the bare module id because during a swap the
10801/// id's active slot is already held by the incumbent: an id-keyed wait would
10802/// report the incumbent's registration as the candidate's and a candidate that
10803/// never registers would look registered. A swap candidate waits on
10804/// `crate::registry::RegistrationSlot::Candidate`.
10805async fn wait_for_slot_registration(
10806    registry: &Registry,
10807    slot: crate::registry::RegistrationSlot<'_>,
10808    module_id: &str,
10809    snapshot: &SharedSnapshot,
10810    child: &mut SupervisedChild,
10811    wait: Duration,
10812) -> Result<RegistrationWaitOutcome, SuperviseError> {
10813    let deadline = Instant::now() + wait;
10814    loop {
10815        if registry
10816            .registration(slot)
10817            .map_err(SuperviseError::Registry)?
10818            .is_some()
10819        {
10820            return Ok(RegistrationWaitOutcome::Registered);
10821        }
10822
10823        let now = Instant::now();
10824        if now >= deadline {
10825            return Ok(RegistrationWaitOutcome::TimedOut);
10826        }
10827        let remaining = deadline.saturating_duration_since(now);
10828        let poll = remaining.min(REGISTRY_RELEASE_POLL);
10829
10830        tokio::select! {
10831            wait_result = child.wait() => {
10832                let status = wait_result.map_err(|source| SuperviseError::Wait {
10833                    module_id: module_id.to_string(),
10834                    source,
10835                })?;
10836                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10837                    snapshot,
10838                    child,
10839                    &status,
10840                )));
10841            }
10842            _ = sleep(poll) => {}
10843        }
10844    }
10845}
10846
10847fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10848    // A replacement process that exits before HELLO did not provide service, even
10849    // if it used status 0. Count it against the restart cap as a new-binary failure.
10850    if exit_report.kind != ExitKind::DeliberateSeverance {
10851        exit_report.kind = ExitKind::Crash;
10852    }
10853    exit_report
10854}
10855
10856async fn handle_reload_child_registration_failure(
10857    spec: &ModuleSpec,
10858    runtime: &SupervisorRuntimeConfig,
10859    registry: &Registry,
10860    process_liveness: &SupervisorProcessLiveness,
10861    snapshot: &SharedSnapshot,
10862    _child: &mut Option<SupervisedChild>,
10863    failure: ReloadRegistrationFailure,
10864) -> Result<(), SuperviseError> {
10865    let ReloadRegistrationFailure {
10866        exit_report,
10867        reason,
10868    } = failure;
10869    match on_child_exit(
10870        spec,
10871        runtime.restart_policy,
10872        registry,
10873        snapshot,
10874        &runtime.terminal_ring,
10875        &runtime.spawn_events,
10876        &runtime.child_roster,
10877        exit_report,
10878        runtime.recovery_is_daemon,
10879    )
10880    .await
10881    {
10882        NextAction::Stop {
10883            registration_released,
10884        } => {
10885            if registration_released {
10886                process_liveness.untrack_if_current(&spec.module_id, snapshot);
10887            }
10888        }
10889        NextAction::Restart { schedule } => {
10890            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10891                schedule.delay
10892            });
10893            if let Some(schedule) = schedule {
10894                log_crash_respawn(&spec.module_id, schedule);
10895            }
10896            schedule_respawn(
10897                runtime,
10898                snapshot,
10899                &spec.module_id,
10900                delay,
10901                RespawnKind::Spawn,
10902                true,
10903            )?;
10904        }
10905    }
10906    Err(SuperviseError::ReloadFailed {
10907        module_id: spec.module_id.clone(),
10908        reason,
10909    })
10910}
10911
10912async fn handle_reload_spawn_failure(
10913    spec: &ModuleSpec,
10914    runtime: &SupervisorRuntimeConfig,
10915    process_liveness: &SupervisorProcessLiveness,
10916    snapshot: &SharedSnapshot,
10917    _child: &mut Option<SupervisedChild>,
10918    reason: String,
10919) -> Result<(), SuperviseError> {
10920    let now = Instant::now();
10921    let mut schedule = None;
10922    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10923        clear_current_process_facts(state);
10924        if state.enabled {
10925            schedule = state.next_crash_restart(&runtime.restart_policy, now);
10926            state.state = if schedule.is_some() {
10927                ModuleState::Restarting
10928            } else {
10929                if runtime
10930                    .failure_context(runtime.recovery_is_daemon)
10931                    .may_retry()
10932                {
10933                    state.failed_retry.schedule(&spec.module_id);
10934                }
10935                ModuleState::Failed
10936            };
10937        } else {
10938            state.state = ModuleState::Disabled;
10939        }
10940    })?;
10941    if let Some(schedule) = schedule {
10942        schedule_respawn(
10943            runtime,
10944            snapshot,
10945            &spec.module_id,
10946            schedule.delay,
10947            RespawnKind::Spawn,
10948            true,
10949        )?;
10950    } else {
10951        process_liveness.untrack_if_current(&spec.module_id, snapshot);
10952    }
10953    Err(SuperviseError::ReloadFailed {
10954        module_id: spec.module_id.clone(),
10955        reason,
10956    })
10957}
10958
10959fn control_flags() -> Flags {
10960    Flags::new(false, Priority::Passive, false)
10961}
10962
10963#[allow(clippy::too_many_arguments)]
10964async fn drain_optional_child(
10965    module_id: &str,
10966    protocol: ModuleProtocol,
10967    stop_notice: StopNotice,
10968    registry: &Registry,
10969    forwarding: Option<&ForwardingTable>,
10970    snapshot: &SharedSnapshot,
10971    terminal_ring: &Arc<Mutex<TerminalRing>>,
10972    spawn_events: &SpawnEventFeed,
10973    child: &mut Option<SupervisedChild>,
10974    drain_timeout: Duration,
10975    final_state: ModuleState,
10976    enabled: Option<bool>,
10977    failure: &FailureContext,
10978) -> Result<(), SuperviseError> {
10979    if let Some(child) = child.take() {
10980        drain_child_to_state(
10981            module_id,
10982            protocol,
10983            stop_notice,
10984            registry,
10985            forwarding,
10986            snapshot,
10987            terminal_ring,
10988            spawn_events,
10989            child,
10990            drain_timeout,
10991            final_state,
10992            enabled,
10993            failure,
10994        )
10995        .await
10996    } else {
10997        update_snapshot(snapshot, Some(module_id), |state| {
10998            state.state = final_state;
10999            if final_state == ModuleState::Failed {
11000                state.failed_retry.failed_at = Some(Instant::now());
11001            }
11002            if let Some(enabled) = enabled {
11003                state.enabled = enabled;
11004            }
11005            clear_current_process_facts(state);
11006        })?;
11007        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11008    }
11009}
11010
11011#[allow(clippy::too_many_arguments)]
11012async fn drain_child_to_state(
11013    module_id: &str,
11014    _protocol: ModuleProtocol,
11015    stop_notice: StopNotice,
11016    registry: &Registry,
11017    forwarding: Option<&ForwardingTable>,
11018    snapshot: &SharedSnapshot,
11019    terminal_ring: &Arc<Mutex<TerminalRing>>,
11020    spawn_events: &SpawnEventFeed,
11021    mut child: SupervisedChild,
11022    drain_timeout: Duration,
11023    final_state: ModuleState,
11024    enabled: Option<bool>,
11025    failure: &FailureContext,
11026) -> Result<(), SuperviseError> {
11027    let protocol = child.protocol;
11028    update_snapshot(snapshot, Some(module_id), |state| {
11029        state.state = ModuleState::Draining;
11030        state.draining_to_replace = final_state == ModuleState::Restarting;
11031        if let Some(enabled) = enabled {
11032            state.enabled = enabled;
11033        }
11034    })?;
11035
11036    // The wait below is the same budget in every case; what differs is
11037    // whether anything has ASKED the child to stop before it starts. Only a
11038    // forwarding drain that reached the module's registered connection has
11039    // (`module.draining`, then a module GOODBYE). Every other child was told
11040    // nothing: a `protocol: "none"` module, which never registers; a subc
11041    // module spawned moments ago that has not sent HELLO yet; or a stop that
11042    // runs no forwarding drain. Without a signal the budget is only a delay
11043    // in front of SIGKILL -- and the not-yet-registered child is the worst
11044    // case, because it registers into a module that is already draining,
11045    // is never told, and is killed while healthy.
11046    if stop_notice != StopNotice::SentOverConnection {
11047        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11048            info!(
11049                module_id,
11050                pid = child.pid,
11051                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11052                "module has no connection yet; requesting stop by signal"
11053            );
11054        }
11055        request_graceful_stop(module_id, &child);
11056    }
11057
11058    let exit_report = match timeout(drain_timeout, child.wait()).await {
11059        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11060        Ok(Err(source)) => {
11061            fail_after_wait_error(
11062                module_id,
11063                snapshot,
11064                terminal_ring,
11065                spawn_events,
11066                format!("failed to wait during drain: {source}"),
11067                failure,
11068            );
11069            return Err(SuperviseError::Wait {
11070                module_id: module_id.to_string(),
11071                source,
11072            });
11073        }
11074        Err(_) => {
11075            // Mirror the sibling arm above: state is already `Draining`, and an
11076            // error propagated from here would strand it there -- a state
11077            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
11078            // `Failed | Stopped`), leaving an operator Restart as the only exit.
11079            // `Failed` before `?` keeps the module operator-visible and
11080            // revivable. Trigger is an ESRCH race (process exits between the
11081            // drain timeout firing and the kill) or a post-kill wait failure
11082            // (issue #34).
11083            //
11084            // Logged because the kill is otherwise visible only as signal 9 in
11085            // the terminal ring, and the budget it follows can be long enough
11086            // that consumers see a stretch of refusals with no stated cause.
11087            warn!(
11088                module_id,
11089                pid = child.pid,
11090                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11091                reason = ?final_state,
11092                ?stop_notice,
11093                "drain budget expired before the module exited; killing it"
11094            );
11095            child.start_kill().map_err(|source| {
11096                fail_after_wait_error(
11097                    module_id,
11098                    snapshot,
11099                    terminal_ring,
11100                    spawn_events,
11101                    format!("failed to kill during drain: {source}"),
11102                    failure,
11103                );
11104                SuperviseError::Kill {
11105                    module_id: module_id.to_string(),
11106                    source,
11107                }
11108            })?;
11109            let status = child.wait().await.map_err(|source| {
11110                fail_after_wait_error(
11111                    module_id,
11112                    snapshot,
11113                    terminal_ring,
11114                    spawn_events,
11115                    format!("failed to wait after drain kill: {source}"),
11116                    failure,
11117                );
11118                SuperviseError::Wait {
11119                    module_id: module_id.to_string(),
11120                    source,
11121                }
11122            })?;
11123            classify_reaped_child_exit(snapshot, &child, &status)
11124        }
11125    };
11126
11127    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11128    publish_after_terminal_record(
11129        module_id,
11130        || {
11131            record_terminal_with_detail(
11132                module_id,
11133                terminal_ring,
11134                spawn_events,
11135                &exit_report,
11136                terminal_disposition(final_state),
11137                detail,
11138            )
11139        },
11140        || {
11141            update_snapshot(snapshot, Some(module_id), |state| {
11142                state.state = final_state;
11143                if final_state == ModuleState::Failed {
11144                    state.failed_retry.failed_at = Some(Instant::now());
11145                }
11146                if let Some(enabled) = enabled {
11147                    state.enabled = enabled;
11148                }
11149                clear_current_process_facts(state);
11150                state.last_exit = Some(exit_report.clone());
11151                if exit_report.kind == ExitKind::DeliberateSeverance {
11152                    state.lifetime_restarts += 1;
11153                }
11154            })
11155        },
11156    )?;
11157    child.drain_stderr(module_id).await;
11158
11159    release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11160}
11161
11162/// Ask a child that nothing else has asked to stop, by signal.
11163///
11164/// A registered subc module is asked over its own connection: the drain sends
11165/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
11166/// module GOODBYE, and the module stops itself. A module that speaks no subc
11167/// wire receives none of that, and neither does a subc module that has not
11168/// registered yet, so for them the drain budget would be pure delay in front of
11169/// a SIGKILL -- and for a process with a store to flush (JetStream is the
11170/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
11171/// into a recovery on the next start.
11172///
11173/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
11174/// rule rather than an optimisation: that module's graceful stop is already
11175/// running by the time its child is drained, and a signal would race it.
11176///
11177/// Best-effort by construction. A child that has already exited is the ordinary
11178/// case rather than an error (the kill lands on a reaped or exiting pid), so a
11179/// failure is logged at debug and the wait-then-kill below still decides the
11180/// outcome.
11181#[cfg(unix)]
11182fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
11183    let Some(pid) = child
11184        .id()
11185        .and_then(|pid| i32::try_from(pid).ok())
11186        .and_then(rustix::process::Pid::from_raw)
11187    else {
11188        debug!(
11189            module_id,
11190            "no pid to signal for teardown; falling through to the drain wait"
11191        );
11192        return;
11193    };
11194    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
11195        Ok(()) => debug!(
11196            module_id,
11197            "sent SIGTERM to a module nothing else asked to stop"
11198        ),
11199        Err(err) => debug!(
11200            module_id,
11201            error = %err,
11202            "SIGTERM to module failed; the drain wait and kill still apply"
11203        ),
11204    }
11205}
11206
11207/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
11208/// Windows does offer need cooperation this supervisor cannot assume: a console
11209/// control event requires sharing a console with the child, and `WM_CLOSE`
11210/// requires the child to pump a message loop. A supervised server process does
11211/// neither, so there is nothing to send and teardown is the wait followed by the
11212/// kill. Emulating a signal here would mean inventing a stop protocol, which is
11213/// the thing `protocol: "none"` exists to avoid.
11214#[cfg(not(unix))]
11215fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
11216    debug!(
11217        module_id,
11218        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
11219    );
11220}
11221
11222fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
11223    match final_state {
11224        ModuleState::Stopped => TerminalDisposition::Stopped,
11225        ModuleState::Disabled => TerminalDisposition::Disabled,
11226        ModuleState::Restarting => TerminalDisposition::Restarting,
11227        ModuleState::Failed => TerminalDisposition::Failed,
11228        ModuleState::Starting
11229        | ModuleState::Running
11230        | ModuleState::Unresponsive
11231        | ModuleState::Draining => {
11232            unreachable!("terminal exits only finish in terminal or restarting states")
11233        }
11234    }
11235}
11236
11237/// Release a reaped child's registration before allowing another spawn.
11238///
11239/// EOF is not a process-lifetime signal: an inherited socket can stay open
11240/// indefinitely, and serial frame dispatch can be waiting on egress instead of
11241/// reading EOF. After the normal release grace, request connection close (which
11242/// cancels both reads and dispatch), then allow one more release grace for the
11243/// connection guard's forwarding cleanup. Never evict a different connection.
11244async fn release_dead_registration(
11245    registry: &Registry,
11246    forwarding: Option<&ForwardingTable>,
11247    snapshot: &SharedSnapshot,
11248    module_id: &str,
11249    failure: &FailureContext,
11250) -> Result<(), SuperviseError> {
11251    let result = async {
11252        let registration = registry
11253            .get_module(module_id)
11254            .map_err(SuperviseError::Registry)?;
11255        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
11256            Ok(()) => return Ok(()),
11257            Err(SuperviseError::RegistrationStillActive { .. }) => {}
11258            Err(err) => return Err(err),
11259        }
11260        let pid = lock_snapshot(snapshot)?.reaped_pid;
11261        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
11262            warn!(
11263                module_id,
11264                pid,
11265                connection_id = registration.connection_id.get(),
11266                "reaped module registration outlived release grace; closing dead connection"
11267            );
11268            forwarding.request_connection_close(
11269                registration.connection_id,
11270                CloseReason::new(
11271                    "supervised_process_reaped",
11272                    format!("module '{module_id}' pid {pid} exited"),
11273                ),
11274            );
11275            wait_for_slot_registration_release(
11276                registry,
11277                crate::registry::RegistrationSlot::Connection(registration.connection_id),
11278                REGISTRY_RELEASE_TIMEOUT,
11279            )
11280            .await?;
11281        }
11282        wait_for_registration_release(registry, module_id, Duration::ZERO).await
11283    }
11284    .await;
11285    if let Err(err) = &result {
11286        fail_snapshot(snapshot, Some(module_id), None, failure);
11287        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
11288    }
11289    result
11290}
11291
11292/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
11293/// plain stop or restart waits for before it spawns a replacement.
11294async fn wait_for_registration_release(
11295    registry: &Registry,
11296    module_id: &str,
11297    wait: Duration,
11298) -> Result<(), SuperviseError> {
11299    wait_for_slot_registration_release(
11300        registry,
11301        crate::registry::RegistrationSlot::Active(module_id),
11302        wait,
11303    )
11304    .await
11305}
11306
11307/// Wait for the registration in `slot` to go away.
11308///
11309/// Keyed on the slot rather than the bare module id because a successful swap
11310/// never empties the id's active slot (the promoted candidate is in it), so an
11311/// id-keyed wait for the incumbent's release would always time out. Draining a
11312/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
11313/// incumbent's connection instead.
11314async fn wait_for_slot_registration_release(
11315    registry: &Registry,
11316    slot: crate::registry::RegistrationSlot<'_>,
11317    wait: Duration,
11318) -> Result<(), SuperviseError> {
11319    let deadline = Instant::now() + wait;
11320    let mut release_events = registration_release_events().subscribe();
11321    let still_active = |registration: &crate::registry::ModuleRegistration| {
11322        SuperviseError::RegistrationStillActive {
11323            module_id: registration.manifest.module_id.clone(),
11324            waited: wait,
11325        }
11326    };
11327    loop {
11328        let _observed_generation = *release_events.borrow_and_update();
11329        let Some(registration) = registry
11330            .registration(slot)
11331            .map_err(SuperviseError::Registry)?
11332        else {
11333            return Ok(());
11334        };
11335
11336        let now = Instant::now();
11337        if now >= deadline {
11338            return Err(still_active(&registration));
11339        }
11340
11341        let remaining = deadline.saturating_duration_since(now);
11342        match timeout(remaining, release_events.changed()).await {
11343            Ok(Ok(())) | Ok(Err(_)) => {}
11344            Err(_) => return Err(still_active(&registration)),
11345        }
11346    }
11347}
11348
11349#[cfg(test)]
11350mod slot_registration_wait_tests {
11351    use super::*;
11352    use crate::registry::{ConnectionId, RegistrationSlot};
11353    use subc_protocol::manifest::ModuleManifest;
11354
11355    #[tokio::test]
11356    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
11357        let registry = Arc::new(Registry::default());
11358        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
11359        let runtime = supervisor.runtime_config();
11360        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
11361        let spec = ModuleSpec {
11362            module_id: "enable-stale-registration".to_string(),
11363            program: PathBuf::from("/missing/enable-retry-test"),
11364            args: Vec::new(),
11365            env: Vec::new(),
11366            reserved: false,
11367            reserved_prefixes: Vec::new(),
11368            protocol: ModuleProtocol::Subc,
11369            overlap: Default::default(),
11370        };
11371        let connection = ConnectionId::new(90);
11372        registry
11373            .register_with_control_ops(
11374                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
11375                1,
11376                connection,
11377                Vec::new(),
11378            )
11379            .unwrap();
11380        let mut child = None;
11381        let err = set_child_enabled(
11382            &spec,
11383            &runtime,
11384            &registry,
11385            &supervisor.process_liveness,
11386            &snapshot,
11387            &mut child,
11388            true,
11389            &runtime.failure_context(false),
11390        )
11391        .await
11392        .unwrap_err();
11393        assert!(matches!(
11394            err,
11395            SuperviseError::RegistrationStillActive { .. }
11396        ));
11397        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11398        assert!(child.is_none());
11399        registry.deregister_connection(connection).unwrap();
11400        let err = set_child_enabled(
11401            &spec,
11402            &runtime,
11403            &registry,
11404            &supervisor.process_liveness,
11405            &snapshot,
11406            &mut child,
11407            true,
11408            &runtime.failure_context(false),
11409        )
11410        .await
11411        .unwrap_err();
11412        assert!(
11413            matches!(err, SuperviseError::Spawn { .. }),
11414            "second enable must attempt a spawn: {err}"
11415        );
11416        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11417    }
11418
11419    const INCUMBENT: u64 = 1;
11420    const CANDIDATE: u64 = 2;
11421
11422    fn swapped_registry() -> Arc<Registry> {
11423        let registry = Arc::new(Registry::default());
11424        let manifest = ModuleManifest::builder("m", "0.1.0").build();
11425        registry
11426            .register_with_control_ops(
11427                manifest.clone(),
11428                1,
11429                ConnectionId::new(INCUMBENT),
11430                Vec::new(),
11431            )
11432            .unwrap();
11433        registry
11434            .register_candidate_with_control_ops(
11435                manifest,
11436                1,
11437                ConnectionId::new(CANDIDATE),
11438                Vec::new(),
11439            )
11440            .unwrap();
11441        registry
11442    }
11443
11444    /// After a promotion the id's active slot is held by the new process, so an
11445    /// id-keyed wait for the incumbent's release can never succeed; the
11446    /// connection-keyed wait completes as soon as the incumbent deregisters.
11447    #[tokio::test]
11448    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11449        let registry = swapped_registry();
11450        registry.promote_candidate("m").unwrap().unwrap();
11451
11452        assert!(matches!(
11453            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
11454            Err(SuperviseError::RegistrationStillActive { .. })
11455        ));
11456
11457        // Still held while the incumbent's connection has not deregistered.
11458        assert!(matches!(
11459            wait_for_slot_registration_release(
11460                &registry,
11461                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11462                Duration::from_millis(50),
11463            )
11464            .await,
11465            Err(SuperviseError::RegistrationStillActive { .. })
11466        ));
11467
11468        let releaser = Arc::clone(&registry);
11469        let release = tokio::spawn(async move {
11470            sleep(Duration::from_millis(20)).await;
11471            releaser
11472                .deregister_connection(ConnectionId::new(INCUMBENT))
11473                .unwrap();
11474            notify_registration_release();
11475        });
11476        wait_for_slot_registration_release(
11477            &registry,
11478            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11479            Duration::from_secs(5),
11480        )
11481        .await
11482        .expect("the incumbent's own registration is released");
11483        release.await.unwrap();
11484        assert!(registry.get_module("m").unwrap().is_some());
11485    }
11486
11487    /// The candidate slot is waited on separately from the active slot: the
11488    /// incumbent's registration neither holds up nor stands in for it.
11489    #[tokio::test]
11490    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11491        let registry = swapped_registry();
11492        assert!(matches!(
11493            wait_for_slot_registration_release(
11494                &registry,
11495                RegistrationSlot::Candidate("m"),
11496                Duration::from_millis(50),
11497            )
11498            .await,
11499            Err(SuperviseError::RegistrationStillActive { .. })
11500        ));
11501        registry
11502            .deregister_connection(ConnectionId::new(CANDIDATE))
11503            .unwrap();
11504        wait_for_slot_registration_release(
11505            &registry,
11506            RegistrationSlot::Candidate("m"),
11507            Duration::from_millis(50),
11508        )
11509        .await
11510        .expect("a candidate slot with no candidate is released");
11511        assert!(registry
11512            .registration(RegistrationSlot::Active("m"))
11513            .unwrap()
11514            .is_some());
11515    }
11516}
11517
11518fn classify_exit(status: &ExitStatus) -> ExitReport {
11519    ExitReport {
11520        kind: if status.success() {
11521            ExitKind::Clean
11522        } else {
11523            ExitKind::Crash
11524        },
11525        code: status.code(),
11526        signal: exit_signal(status),
11527        at_ms: unix_ms_now(),
11528    }
11529}
11530
11531/// The terminal record for a module whose `wait()` call itself errored (e.g. the
11532/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
11533/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
11534/// disposition still must be `Failed` so the terminal ring is not silently missing
11535/// an entry, matching what `fail_snapshot` records for this same arm.
11536fn wait_error_exit_report() -> ExitReport {
11537    ExitReport {
11538        kind: ExitKind::Crash,
11539        code: None,
11540        signal: None,
11541        at_ms: unix_ms_now(),
11542    }
11543}
11544
11545#[cfg(unix)]
11546fn exit_signal(status: &ExitStatus) -> Option<i32> {
11547    use std::os::unix::process::ExitStatusExt;
11548
11549    status.signal()
11550}
11551
11552#[cfg(not(unix))]
11553fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11554    None
11555}
11556
11557/// Give an operator-touched module its full crash budget back.
11558///
11559/// Named for the counter it used to zero; it now empties the in-window ring,
11560/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
11561/// ledger of what happened survives every operator action.
11562fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11563    update_snapshot(snapshot, Some(module_id), |state| {
11564        state.clear_crash_restarts();
11565    })
11566}
11567
11568fn set_running(
11569    snapshot: &SharedSnapshot,
11570    child: &SupervisedChild,
11571    module_id: &str,
11572    spawn_events: &SpawnEventFeed,
11573) -> Result<(), SuperviseError> {
11574    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11575        module_id: Some(module_id.to_string()),
11576    })?;
11577    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11578    if std::mem::take(&mut state.coalesced_restart_pending) {
11579        let generation = state.spawn_generation;
11580        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11581    }
11582    state.drain_disposition_detail = None;
11583    state.spawn_failure = None;
11584    state.waiting_for_previous_pid = None;
11585    state.waiting_for_previous_detail = None;
11586    // Every caller of this is a plain spawn, which always uses the primary key;
11587    // a promoted swap candidate sets the flag itself after this returns.
11588    state.in_alternate_slot = false;
11589    state.configuration_updated_since_spawn = false;
11590    state.spawned_protocol = Some(child.protocol);
11591    state.state = ModuleState::Running;
11592    state.enabled = true;
11593    state.process_alive = true;
11594    state.pid = child.id();
11595    // Probe samples belong to one process. Keep module-level escalation and
11596    // late-answer history, but never charge a new child for its predecessor's misses.
11597    state.health.status = SupervisorHealthStatus::Unknown;
11598    state.health.consecutive_failures = 0;
11599    state.health.detail = None;
11600    state.health.metrics = None;
11601    state.health.last_probe_ms = None;
11602    state.failed_retry.healthy_since = None;
11603    state.failed_retry.failed_at = None;
11604    state.failed_retry.running_since = Some(child.spawned_at);
11605    #[cfg(windows)]
11606    {
11607        state.pipe_consumed = child
11608            .nonce_delivery
11609            .as_ref()
11610            .map(|delivery| delivery.consumed());
11611    }
11612    #[cfg(target_os = "macos")]
11613    {
11614        state.report_ready = Some(Arc::clone(&child.report_ready));
11615    }
11616    state.spawned_at_ms = Some(child.spawned_at_ms);
11617    state.exec_outcome = Some(Arc::clone(&child.exec.0));
11618    state.spawned_from = Some(child.spawned_from.clone());
11619    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11620    state.spawned_file_identity = child.spawned_file_identity.clone();
11621    state.process_start_time = child.process_start_time;
11622    Ok(())
11623}
11624
11625fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11626    state.failed_retry.healthy_since = None;
11627    state.failed_retry.running_since = None;
11628    state.waiting_for_previous_pid = None;
11629    state.waiting_for_previous_detail = None;
11630    state.process_alive = false;
11631    state.spawned_protocol = None;
11632    state.pid = None;
11633    #[cfg(windows)]
11634    {
11635        state.pipe_consumed = None;
11636    }
11637    #[cfg(target_os = "macos")]
11638    {
11639        state.report_ready = None;
11640    }
11641    state.spawned_at_ms = None;
11642    state.exec_outcome = None;
11643    state.spawned_from = None;
11644    state.launch_spec_digest = None;
11645    state.spawned_file_identity = None;
11646    state.process_start_time = None;
11647    state.deliberate_severance = None;
11648}
11649
11650#[cfg(test)]
11651fn record_deliberate_severance(
11652    snapshot: &SharedSnapshot,
11653    identity: ProcessIdentity,
11654) -> Result<(), SuperviseError> {
11655    update_snapshot(snapshot, None, |state| {
11656        state.deliberate_severance = Some(identity);
11657    })
11658}
11659
11660fn apply_deliberate_severance_marker(
11661    snapshot: &SharedSnapshot,
11662    exited_identity: Option<ProcessIdentity>,
11663    mut exit_report: ExitReport,
11664) -> ExitReport {
11665    let marker = lock_snapshot(snapshot)
11666        .ok()
11667        .and_then(|mut state| state.deliberate_severance.take());
11668    if marker.is_some() && marker == exited_identity {
11669        exit_report.kind = ExitKind::DeliberateSeverance;
11670    }
11671    exit_report
11672}
11673
11674fn classify_reaped_child_exit(
11675    snapshot: &SharedSnapshot,
11676    child: &SupervisedChild,
11677    status: &ExitStatus,
11678) -> ExitReport {
11679    let _ = update_snapshot(snapshot, None, |state| {
11680        state.reaped_pid = Some(child.pid);
11681        state.spawn_failure = child.spawn_failure.clone();
11682    });
11683    apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11684}
11685
11686fn fail_snapshot(
11687    snapshot: &SharedSnapshot,
11688    module_id: Option<&str>,
11689    last_exit: Option<ExitReport>,
11690    failure: &FailureContext,
11691) {
11692    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11693        state.state = ModuleState::Failed;
11694        clear_current_process_facts(state);
11695        if failure.may_retry() && state.enabled {
11696            state.failed_retry.schedule(module_id.unwrap_or("unknown"));
11697        } else if failure.shutdown.is_set() {
11698            state.failed_retry.cancel();
11699        }
11700        if let Some(last_exit) = last_exit {
11701            state.last_exit = Some(last_exit);
11702        }
11703    }) {
11704        error!(error = %err, "failed to mark supervisor state failed");
11705    }
11706}
11707
11708fn update_snapshot(
11709    snapshot: &SharedSnapshot,
11710    module_id: Option<&str>,
11711    update: impl FnOnce(&mut SupervisorSnapshot),
11712) -> Result<(), SuperviseError> {
11713    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11714        module_id: module_id.map(ToOwned::to_owned),
11715    })?;
11716    update(&mut state);
11717    Ok(())
11718}
11719
11720const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11721
11722fn lock_snapshot_for_control<'a>(
11723    snapshot: &'a SharedSnapshot,
11724    module_id: &str,
11725    caller: &'static str,
11726) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11727    let started_at = Instant::now();
11728    let guard = lock_snapshot(snapshot)?;
11729    let waited = started_at.elapsed();
11730    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11731        warn!(
11732            module_id = %module_id,
11733            waited_ms = waited.as_millis() as u64,
11734            caller = %caller,
11735            "slow snapshot lock"
11736        );
11737    }
11738    Ok(guard)
11739}
11740
11741fn lock_snapshot(
11742    snapshot: &SharedSnapshot,
11743) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11744    snapshot
11745        .lock()
11746        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11747}
11748
11749#[cfg(test)]
11750mod terminal_history_tests {
11751    use std::{
11752        path::PathBuf,
11753        sync::Arc,
11754        time::{Duration, Instant},
11755    };
11756
11757    use tokio::{sync::mpsc, time::sleep};
11758
11759    use super::{
11760        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11761        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11762        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11763        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11764        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11765        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11766        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11767        SupervisorHealthStatus, SupervisorSnapshot,
11768    };
11769    // The supervisor's clock, distinct from the `std::time::Instant` these tests
11770    // use for their own wall-clock deadlines: crash-restart instants must be on
11771    // the same clock the production code stamps them with, which is tokio's (and
11772    // is what `start_paused` tests can move).
11773    use super::Instant as ClockInstant;
11774    use crate::{
11775        registry::Registry,
11776        terminal_ring::{TerminalRing, TerminalRingConfig},
11777    };
11778    use std::sync::Mutex;
11779    use subc_control::TerminalDisposition;
11780
11781    /// See the twin in `control.rs` for why this derives the path from
11782    /// `current_exe()` and why the existence check is here: `--lib` alone does
11783    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
11784    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
11785    pub(super) fn fake_aft_stub_path() -> PathBuf {
11786        let mut path = std::env::current_exe().expect("current_exe available in tests");
11787        path.pop();
11788        path.pop();
11789        path.push(if cfg!(windows) {
11790            "fake-aft-stub.exe"
11791        } else {
11792            "fake-aft-stub"
11793        });
11794        assert!(
11795            path.exists(),
11796            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11797             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11798            path.display()
11799        );
11800        path
11801    }
11802
11803    #[test]
11804    fn reserved_never_spawned_refuses_every_hello() {
11805        // The canary hole: a reserved id whose module has never spawned had NO
11806        // gate entry and admitted anyone -- the reservation protected the nonce
11807        // holder, not the NAME. Now the entry is present with no legitimate
11808        // holder and refuses all comers.
11809        let supervisor = SupervisorHandle::default();
11810        supervisor.apply_identity_configuration(&ModuleSpec {
11811            module_id: "never-spawned".to_string(),
11812            program: PathBuf::from("/usr/bin/false"),
11813            args: Vec::new(),
11814            env: Vec::new(),
11815            reserved: true,
11816            reserved_prefixes: Vec::new(),
11817            protocol: ModuleProtocol::Subc,
11818            overlap: Default::default(),
11819        });
11820        assert!(
11821            supervisor
11822                .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
11823                .is_err(),
11824            "forged nonce must refuse on a reserved never-spawned id"
11825        );
11826        assert!(
11827            supervisor
11828                .reserved_hello_verification("never-spawned", None)
11829                .is_err(),
11830            "absent nonce must refuse on a reserved never-spawned id"
11831        );
11832        // And a real spawn nonce minted later admits exactly that nonce.
11833        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11834        supervisor.apply_identity_configuration(&ModuleSpec {
11835            module_id: "never-spawned".to_string(),
11836            program: PathBuf::from("/usr/bin/false"),
11837            args: Vec::new(),
11838            env: Vec::new(),
11839            reserved: true,
11840            reserved_prefixes: Vec::new(),
11841            protocol: ModuleProtocol::Subc,
11842            overlap: Default::default(),
11843        });
11844        assert!(supervisor
11845            .reserved_hello_verification("never-spawned", Some("minted"))
11846            .is_ok());
11847        assert!(supervisor
11848            .reserved_hello_verification("never-spawned", Some("forged"))
11849            .is_err());
11850    }
11851
11852    /// Put `count` crash restarts on a snapshot's ring as if they had all just
11853    /// happened, which is what "spent budget" looks like to every reader.
11854    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11855        let now = ClockInstant::now();
11856        for _ in 0..count {
11857            state.crash_restarts.push_back(now);
11858        }
11859    }
11860
11861    /// Age the oldest recorded restart out of `window`, standing in for the hours
11862    /// that would otherwise have to pass. Injecting the instant is the point: a
11863    /// test that slept a real window would take ten minutes and still prove less.
11864    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11865        let aged = state
11866            .crash_restarts
11867            .front()
11868            .expect("a crash restart must be recorded before it can be aged")
11869            .checked_sub(window + Duration::from_secs(1))
11870            .expect("the test clock is far enough from its origin to age an instant");
11871        state.crash_restarts[0] = aged;
11872    }
11873
11874    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11875        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11876        seed_crash_restarts(&mut state, count);
11877        state
11878    }
11879
11880    #[test]
11881    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11882        let policy = RestartPolicy::new(3, Duration::ZERO);
11883        let now = ClockInstant::now();
11884        assert!(daemon_will_restart(
11885            &mut snapshot_with_restarts(true, 2),
11886            &policy,
11887            now
11888        ));
11889        assert!(!daemon_will_restart(
11890            &mut snapshot_with_restarts(true, 3),
11891            &policy,
11892            now
11893        ));
11894        assert!(!daemon_will_restart(
11895            &mut snapshot_with_restarts(false, 0),
11896            &policy,
11897            now
11898        ));
11899    }
11900
11901    #[test]
11902    fn crash_restart_backoff_escalates_with_in_window_count() {
11903        let policy = RestartPolicy::new(4, Duration::from_millis(100))
11904            .with_max_backoff(Duration::from_secs(30));
11905        let now = ClockInstant::now();
11906        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11907        let schedules = (0..4)
11908            .map(|_| {
11909                state
11910                    .next_crash_restart(&policy, now)
11911                    .expect("the test policy allows four crash restarts")
11912            })
11913            .collect::<Vec<_>>();
11914
11915        assert_eq!(
11916            schedules
11917                .iter()
11918                .map(|schedule| schedule.restart_in_window)
11919                .collect::<Vec<_>>(),
11920            vec![0, 1, 2, 3]
11921        );
11922        assert_eq!(
11923            schedules
11924                .iter()
11925                .map(|schedule| schedule.delay)
11926                .collect::<Vec<_>>(),
11927            vec![
11928                Duration::from_millis(100),
11929                Duration::from_secs(1),
11930                Duration::from_secs(10),
11931                Duration::from_secs(30),
11932            ]
11933        );
11934    }
11935
11936    #[test]
11937    fn crash_restart_backoff_resets_after_ring_clear() {
11938        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11939        let now = ClockInstant::now();
11940        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11941        assert_eq!(
11942            state.next_crash_restart(&policy, now).unwrap().delay,
11943            Duration::from_millis(100)
11944        );
11945        assert_eq!(
11946            state.next_crash_restart(&policy, now).unwrap().delay,
11947            Duration::from_secs(1)
11948        );
11949
11950        state.clear_crash_restarts();
11951        let schedule = state
11952            .next_crash_restart(&policy, now)
11953            .expect("a cleared ring must allow another restart");
11954        assert_eq!(schedule.restart_in_window, 0);
11955        assert_eq!(schedule.delay, Duration::from_millis(100));
11956    }
11957
11958    #[test]
11959    fn crash_restart_backoff_ignores_aged_restarts() {
11960        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11961        let now = ClockInstant::now();
11962        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11963        state
11964            .next_crash_restart(&policy, now)
11965            .expect("the first restart is allowed");
11966        state
11967            .next_crash_restart(&policy, now)
11968            .expect("the second restart is allowed");
11969        state.crash_restarts[0] = now
11970            .checked_sub(policy.window + Duration::from_secs(1))
11971            .expect("the fake clock can age a restart past the window");
11972
11973        let schedule = state
11974            .next_crash_restart(&policy, now)
11975            .expect("an aged restart must release its slot");
11976        assert_eq!(schedule.restart_in_window, 1);
11977        assert_eq!(schedule.delay, Duration::from_secs(1));
11978        assert_eq!(state.crash_restarts.len(), 2);
11979    }
11980
11981    /// The budget is a rate: the same three spent restarts refuse a respawn
11982    /// while they are recent and allow one once they have aged past the window.
11983    /// Nothing about the module changed in between, which is the whole point.
11984    #[test]
11985    fn a_budget_spent_before_the_window_no_longer_refuses() {
11986        let policy = RestartPolicy::new(3, Duration::ZERO);
11987        let mut state = snapshot_with_restarts(true, 3);
11988        let now = ClockInstant::now();
11989        assert!(!daemon_will_restart(&mut state, &policy, now));
11990
11991        assert!(daemon_will_restart(
11992            &mut state,
11993            &policy,
11994            now + policy.window + Duration::from_secs(1)
11995        ));
11996        assert!(
11997            state.crash_restarts.is_empty(),
11998            "reading the budget must drop the instants that left the window"
11999        );
12000    }
12001
12002    fn module_with_recovery_snapshot(
12003        state: ModuleState,
12004        enabled: bool,
12005        restart_count: u32,
12006    ) -> SupervisedModule {
12007        let registry = Arc::new(Registry::default());
12008        let supervisor =
12009            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
12010        let runtime = supervisor.runtime_config();
12011        let spec = ModuleSpec {
12012            module_id: "recovery-snapshot".to_string(),
12013            program: fake_aft_stub_path(),
12014            args: Vec::new(),
12015            env: Vec::new(),
12016            reserved: false,
12017            reserved_prefixes: Vec::new(),
12018            protocol: ModuleProtocol::Subc,
12019            overlap: Default::default(),
12020        };
12021        let mut snapshot = SupervisorSnapshot::new(state, enabled);
12022        seed_crash_restarts(&mut snapshot, restart_count);
12023        // These tests read synthetic snapshots. A real child and monitor would
12024        // race those reads by replacing the requested state during startup.
12025        let (commands, _rx) = mpsc::channel(4);
12026        SupervisedModule {
12027            inner: Arc::new(SupervisedModuleInner {
12028                module_id: spec.module_id.clone(),
12029                registry,
12030                snapshot: Arc::new(Mutex::new(snapshot)),
12031                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12032                    spec,
12033                    health: runtime.health,
12034                })),
12035                stderr_ring: runtime.stderr_ring,
12036                terminal_ring: runtime.terminal_ring,
12037                commands,
12038                monitor: Mutex::new(None),
12039                restart_policy: runtime.restart_policy,
12040                effective_drain_timeout: runtime.effective_drain_timeout,
12041                provenance_probe: supervisor.provenance_probe.clone(),
12042            }),
12043        }
12044    }
12045
12046    #[cfg(target_os = "linux")]
12047    #[tokio::test]
12048    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12049        let supervisor =
12050            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12051                .with_cgroup_placement(None);
12052        let result = supervisor.spawn(ModuleSpec {
12053            module_id: "no-cgroup-placement".to_string(),
12054            program: fake_aft_stub_path(),
12055            args: Vec::new(),
12056            env: Vec::new(),
12057            reserved: false,
12058            reserved_prefixes: Vec::new(),
12059            protocol: ModuleProtocol::Subc,
12060            overlap: Default::default(),
12061        });
12062
12063        assert!(
12064            result.is_ok(),
12065            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12066        );
12067    }
12068
12069    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12070    async fn undecided_snapshot_uses_shared_restart_predicate() {
12071        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12072            .will_recover_after_connection_loss()
12073            .unwrap());
12074        assert!(
12075            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12076                .will_recover_after_connection_loss()
12077                .unwrap()
12078        );
12079    }
12080
12081    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12082    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12083        assert!(
12084            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12085                .will_recover_after_connection_loss()
12086                .unwrap()
12087        );
12088    }
12089
12090    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12091    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12092        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12093            .will_recover_after_connection_loss()
12094            .unwrap());
12095        assert!(
12096            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12097                .will_recover_after_connection_loss()
12098                .unwrap()
12099        );
12100    }
12101
12102    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12103    async fn warming_snapshot_is_limited_to_startup_phases() {
12104        for state in [
12105            ModuleState::Starting,
12106            ModuleState::Running,
12107            ModuleState::Restarting,
12108        ] {
12109            assert!(
12110                module_with_recovery_snapshot(state, true, 0)
12111                    .is_warming()
12112                    .unwrap(),
12113                "{state:?} should be warming"
12114            );
12115        }
12116        for state in [
12117            ModuleState::Unresponsive,
12118            ModuleState::Draining,
12119            ModuleState::Stopped,
12120            ModuleState::Failed,
12121            ModuleState::Disabled,
12122        ] {
12123            assert!(
12124                !module_with_recovery_snapshot(state, true, 0)
12125                    .is_warming()
12126                    .unwrap(),
12127                "{state:?} should not be warming"
12128            );
12129        }
12130    }
12131
12132    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12133    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12134        let registry = Arc::new(Registry::default());
12135        let supervisor =
12136            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
12137        let module = supervisor
12138            .spawn(ModuleSpec {
12139                module_id: "terminal-history".to_string(),
12140                program: fake_aft_stub_path(),
12141                args: Vec::new(),
12142                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12143                reserved: false,
12144                reserved_prefixes: Vec::new(),
12145                protocol: ModuleProtocol::Subc,
12146                overlap: Default::default(),
12147            })
12148            .unwrap();
12149
12150        let deadline = Instant::now() + Duration::from_secs(5);
12151        loop {
12152            let history = module.terminal_history();
12153            if history.entries.len() == 2 {
12154                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
12155                assert_eq!(history.dropped, 0);
12156                assert_eq!(
12157                    history
12158                        .entries
12159                        .iter()
12160                        .map(|entry| entry.exit_code)
12161                        .collect::<Vec<_>>(),
12162                    vec![Some(23), Some(23)]
12163                );
12164                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
12165                return;
12166            }
12167            assert!(
12168                Instant::now() < deadline,
12169                "module did not retain two terminal exits: {history:?}"
12170            );
12171            sleep(Duration::from_millis(10)).await;
12172        }
12173    }
12174
12175    /// A disable issued while a crash respawn is still backing off must preempt
12176    /// that respawn: the operator's stop wins, the disable must not queue behind
12177    /// the backoff, and the module must never come back up afterwards.
12178    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12179    async fn disable_during_crash_backoff_cancels_pending_respawn() {
12180        let backoff = Duration::from_secs(2);
12181        let supervisor = Supervisor::new_for_test(
12182            Arc::new(Registry::default()),
12183            RestartPolicy::new(10, backoff),
12184        );
12185        let module = supervisor
12186            .spawn(ModuleSpec {
12187                module_id: "disable-during-backoff".to_string(),
12188                program: fake_aft_stub_path(),
12189                args: Vec::new(),
12190                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12191                reserved: false,
12192                reserved_prefixes: Vec::new(),
12193                protocol: ModuleProtocol::Subc,
12194                overlap: Default::default(),
12195            })
12196            .unwrap();
12197
12198        // Wait for the first crash to put the module into its backoff window.
12199        let deadline = Instant::now() + Duration::from_secs(5);
12200        loop {
12201            if module.status().unwrap().state == ModuleState::Restarting {
12202                break;
12203            }
12204            assert!(
12205                Instant::now() < deadline,
12206                "module never entered the crash backoff"
12207            );
12208            sleep(Duration::from_millis(10)).await;
12209        }
12210
12211        let started = Instant::now();
12212        module.set_enabled(false).await.unwrap();
12213        let waited = started.elapsed();
12214
12215        assert!(
12216            waited < backoff / 2,
12217            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
12218        );
12219        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
12220
12221        // Outlast the backoff: the respawn it was counting down to must never run.
12222        sleep(backoff + Duration::from_millis(500)).await;
12223        let status = module.status().unwrap();
12224        assert_eq!(status.state, ModuleState::Disabled);
12225        assert_eq!(
12226            status.spawn_generation, 1,
12227            "module respawned after the operator disabled it"
12228        );
12229    }
12230
12231    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
12232    /// the shape of nats-server, the program this rule exists for.
12233    #[cfg(unix)]
12234    fn protocol_none_sigterm_exits_clean_spec(
12235        module_id: &str,
12236        dir: &std::path::Path,
12237    ) -> (ModuleSpec, PathBuf, PathBuf) {
12238        let ready = dir.join("ready");
12239        let marker = dir.join("sigterm");
12240        let spec = ModuleSpec {
12241            module_id: module_id.to_string(),
12242            program: fake_aft_stub_path(),
12243            args: Vec::new(),
12244            env: vec![
12245                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
12246                (
12247                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
12248                    marker.display().to_string(),
12249                ),
12250                (
12251                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
12252                    ready.display().to_string(),
12253                ),
12254            ],
12255            reserved: false,
12256            reserved_prefixes: Vec::new(),
12257            protocol: ModuleProtocol::None,
12258            overlap: Default::default(),
12259        };
12260        (spec, ready, marker)
12261    }
12262
12263    /// Wait for a file the child writes, so a signal is never sent before the
12264    /// child's SIGTERM handler is installed (the default disposition would
12265    /// kill it by signal and the exit would not be clean).
12266    #[cfg(unix)]
12267    async fn wait_for_file(path: &std::path::Path) {
12268        let deadline = Instant::now() + Duration::from_secs(10);
12269        while !path.exists() {
12270            assert!(
12271                Instant::now() < deadline,
12272                "{} never appeared",
12273                path.display()
12274            );
12275            sleep(Duration::from_millis(10)).await;
12276        }
12277    }
12278
12279    /// A protocol-none module that exits 0 because something OUTSIDE the
12280    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
12281    /// the crash-path disposition rather than `stopped`.
12282    #[cfg(unix)]
12283    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12284    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
12285        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
12286        let (spec, ready, marker) =
12287            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
12288        let supervisor = Supervisor::new_for_test(
12289            Arc::new(Registry::default()),
12290            RestartPolicy::new(3, Duration::ZERO),
12291        );
12292        let module = supervisor.spawn(spec).unwrap();
12293        wait_for_file(&ready).await;
12294        // The ready file proves the child installed its SIGTERM handler, not
12295        // that the supervisor has processed the privacy trampoline's exec
12296        // acknowledgement. On macOS status withholds the pid until then.
12297        let deadline = Instant::now() + Duration::from_secs(10);
12298        let first_pid = loop {
12299            let status = module.status().unwrap();
12300            if status.state == ModuleState::Running {
12301                if let Some(pid) = status.pid {
12302                    break pid;
12303                }
12304            }
12305            assert!(
12306                Instant::now() < deadline,
12307                "a running module must report its pid after exec confirmation: {status:?}"
12308            );
12309            sleep(Duration::from_millis(10)).await;
12310        };
12311
12312        rustix::process::kill_process(
12313            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
12314            rustix::process::Signal::TERM,
12315        )
12316        .unwrap();
12317
12318        let deadline = Instant::now() + Duration::from_secs(10);
12319        let respawned = loop {
12320            let status = module.status().unwrap();
12321            if status.state == ModuleState::Running
12322                && status.pid.is_some_and(|pid| pid != first_pid)
12323            {
12324                break status;
12325            }
12326            assert!(
12327                Instant::now() < deadline,
12328                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
12329            );
12330            sleep(Duration::from_millis(10)).await;
12331        };
12332        assert_eq!(respawned.spawn_generation, 2);
12333        assert!(
12334            marker.exists(),
12335            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
12336        );
12337
12338        let history = module.terminal_history();
12339        assert_eq!(history.entries.len(), 1, "{history:?}");
12340        let entry = &history.entries[0];
12341        assert_eq!(entry.exit_code, Some(0));
12342        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
12343        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
12344
12345        module.stop().await.unwrap();
12346    }
12347
12348    /// Repeated unrequested clean exits of a protocol-none module spend the
12349    /// restart budget exactly as crashes do, and the module ends `failed` with
12350    /// the budget named.
12351    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12352    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
12353        let supervisor = Supervisor::new_for_test(
12354            Arc::new(Registry::default()),
12355            RestartPolicy::new(1, Duration::ZERO),
12356        );
12357        let module = supervisor
12358            .spawn(ModuleSpec {
12359                module_id: "none-clean-exit-budget".to_string(),
12360                program: fake_aft_stub_path(),
12361                args: Vec::new(),
12362                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12363                reserved: false,
12364                reserved_prefixes: Vec::new(),
12365                protocol: ModuleProtocol::None,
12366                overlap: Default::default(),
12367            })
12368            .unwrap();
12369
12370        // Failed follows the terminal write; this deadline only bounds a hang,
12371        // not an assumed duration for the two launches or their exit recording.
12372        let deadline = Instant::now() + Duration::from_secs(10);
12373        loop {
12374            let status = module.status().unwrap();
12375            if status.state == ModuleState::Failed {
12376                break;
12377            }
12378            assert!(
12379                Instant::now() < deadline,
12380                "module never exhausted its budget: {status:?} {:?}",
12381                module.terminal_history()
12382            );
12383            sleep(Duration::from_millis(10)).await;
12384        }
12385        let history = module.terminal_history();
12386        assert_eq!(
12387            history
12388                .entries
12389                .iter()
12390                .map(|entry| (entry.exit_code, entry.disposition.clone()))
12391                .collect::<Vec<_>>(),
12392            vec![
12393                (Some(0), TerminalDisposition::Restarting),
12394                (Some(0), TerminalDisposition::Failed),
12395            ]
12396        );
12397        let detail = history.entries[1]
12398            .disposition_detail
12399            .as_deref()
12400            .expect("a budget failure names the budget");
12401        assert!(detail.contains("max_restarts=1"), "{detail}");
12402        assert_eq!(module.status().unwrap().spawn_generation, 2);
12403    }
12404
12405    #[test]
12406    fn restart_budget_failure_is_published_after_its_terminal_record() {
12407        use std::io::{Read, Write};
12408        let supervisor = Supervisor::new_for_test(
12409            Arc::new(Registry::default()),
12410            RestartPolicy::new(0, Duration::ZERO),
12411        );
12412        let runtime = supervisor.runtime_config();
12413        let spec = ModuleSpec {
12414            protocol: ModuleProtocol::None,
12415            ..windowed_crash_spec("budget-publication-order")
12416        };
12417        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12418            ModuleState::Running,
12419            true,
12420        )));
12421        lock_snapshot(&snapshot).unwrap().process_alive = true;
12422        let reaped_snapshot = snapshot.clone();
12423        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12424        let history = runtime.terminal_ring.clone();
12425        let reap = std::thread::spawn(move || {
12426            tokio::runtime::Builder::new_current_thread()
12427                .enable_all()
12428                .build()
12429                .unwrap()
12430                .block_on(on_child_exit(
12431                    &spec,
12432                    runtime.restart_policy,
12433                    &supervisor.registry,
12434                    &reaped_snapshot,
12435                    &runtime.terminal_ring,
12436                    &runtime.spawn_events,
12437                    &runtime.child_roster,
12438                    ExitReport {
12439                        kind: ExitKind::Clean,
12440                        code: Some(0),
12441                        signal: None,
12442                        at_ms: 1,
12443                    },
12444                    true,
12445                ))
12446        });
12447        let mut reached = [0];
12448        barrier.read_exact(&mut reached).unwrap();
12449        assert_eq!(&reached, b"R");
12450        let (before_state, before_alive) = {
12451            let state = lock_snapshot(&snapshot).unwrap();
12452            (state.state, state.process_alive)
12453        };
12454        let before_history = history.lock().unwrap().snapshot();
12455        barrier.write_all(b"X").unwrap();
12456        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12457        assert!(before_history.entries.is_empty());
12458        assert_ne!(
12459            before_state,
12460            ModuleState::Failed,
12461            "Failed was visible before its terminal record could be written"
12462        );
12463        assert!(before_alive);
12464        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12465        let history = history.lock().unwrap().snapshot();
12466        assert_eq!(history.entries.len(), 1);
12467        assert_eq!(history.entries[0].exit_code, Some(0));
12468        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12469    }
12470
12471    #[test]
12472    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12473        use std::io::{Read, Write};
12474
12475        let supervisor = Supervisor::new_for_test(
12476            Arc::new(Registry::default()),
12477            RestartPolicy::new(3, Duration::ZERO),
12478        );
12479        let mut runtime = supervisor.runtime_config();
12480        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12481        let snapshot = module.inner.snapshot.clone();
12482        lock_snapshot(&snapshot).unwrap().process_alive = true;
12483        runtime.terminal_ring = module.inner.terminal_ring.clone();
12484        let spec = windowed_crash_spec(module.module_id());
12485        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12486        let reaped_snapshot = snapshot.clone();
12487        let reap = std::thread::spawn(move || {
12488            tokio::runtime::Builder::new_current_thread()
12489                .enable_all()
12490                .build()
12491                .unwrap()
12492                .block_on(on_child_exit(
12493                    &spec,
12494                    runtime.restart_policy,
12495                    &supervisor.registry,
12496                    &reaped_snapshot,
12497                    &runtime.terminal_ring,
12498                    &runtime.spawn_events,
12499                    &runtime.child_roster,
12500                    ExitReport {
12501                        kind: ExitKind::Crash,
12502                        code: Some(2),
12503                        signal: None,
12504                        at_ms: 1,
12505                    },
12506                    true,
12507                ))
12508        });
12509
12510        // Hold the crash handler between its budget decision and publication.
12511        // Connection cleanup must still see the recovery that this crash chose,
12512        // even when that recovery consumes the last available restart slot.
12513        let mut reached = [0];
12514        barrier.read_exact(&mut reached).unwrap();
12515        assert_eq!(&reached, b"R");
12516        let before_status = module.status().unwrap();
12517        let before_history = module.terminal_history();
12518        let before_recovers = module.will_recover_after_connection_loss().unwrap();
12519        barrier.write_all(b"X").unwrap();
12520        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12521
12522        assert_eq!(before_status.state, ModuleState::Running);
12523        assert!(before_status.process_alive);
12524        assert_eq!(before_status.restart_count, 3);
12525        assert!(before_history.entries.is_empty());
12526        assert!(
12527            before_recovers,
12528            "connection cleanup would report a terminal close for a scheduled crash restart"
12529        );
12530        let restarting = module.status().unwrap();
12531        assert_eq!(restarting.state, ModuleState::Restarting);
12532        assert_eq!(restarting.restart_count, 3);
12533        assert!(module.will_recover_after_connection_loss().unwrap());
12534        let history = module.terminal_history();
12535        assert_eq!(history.entries.len(), 1);
12536        assert_eq!(
12537            history.entries[0].disposition,
12538            TerminalDisposition::Restarting
12539        );
12540
12541        // Once the replacement is running, its next crash must be judged
12542        // against the exhausted budget rather than the previous recovery.
12543        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12544            state.state = ModuleState::Running;
12545            state.respawn_pending = false;
12546        })
12547        .unwrap();
12548        assert!(!module.will_recover_after_connection_loss().unwrap());
12549        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12550            state.enabled = false;
12551            state.respawn_pending = true;
12552        })
12553        .unwrap();
12554        assert!(!module.will_recover_after_connection_loss().unwrap());
12555    }
12556
12557    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12558        use std::net::{TcpListener, TcpStream};
12559        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12560        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12561        let (barrier, _) = listener.accept().unwrap();
12562        for stream in [&writer, &barrier] {
12563            stream
12564                .set_read_timeout(Some(Duration::from_secs(10)))
12565                .unwrap();
12566            stream
12567                .set_write_timeout(Some(Duration::from_secs(10)))
12568                .unwrap();
12569        }
12570        *events.1.lock().unwrap() = Some(writer);
12571        barrier
12572    }
12573
12574    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12575        use std::io::{Read, Write};
12576        let barrier = events.1.lock().unwrap().take();
12577        if let Some(mut barrier) = barrier {
12578            barrier.write_all(b"R").unwrap();
12579            let mut release = [0];
12580            barrier.read_exact(&mut release).unwrap();
12581            assert_eq!(&release, b"X");
12582        }
12583    }
12584
12585    #[test]
12586    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12587        use std::io::{Read, Write};
12588
12589        let supervisor = Supervisor::new_for_test(
12590            Arc::new(Registry::default()),
12591            RestartPolicy::new(0, Duration::ZERO),
12592        );
12593        let mut runtime = supervisor.runtime_config();
12594        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12595        let snapshot = module.inner.snapshot.clone();
12596        runtime.terminal_ring = module.inner.terminal_ring.clone();
12597        let spec = ModuleSpec {
12598            module_id: module.module_id().into(),
12599            program: fake_aft_stub_path(),
12600            protocol: ModuleProtocol::None,
12601            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12602            ..windowed_crash_spec(module.module_id())
12603        };
12604        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12605        let reap = std::thread::spawn(move || {
12606            tokio::runtime::Builder::new_current_thread()
12607                .enable_all()
12608                .build()
12609                .unwrap()
12610                .block_on(async {
12611                    let mut child =
12612                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12613                    health_restart_child(
12614                        &spec,
12615                        &runtime,
12616                        &supervisor.registry,
12617                        &supervisor.process_liveness,
12618                        &snapshot,
12619                        &mut child,
12620                        SupervisorHealthStatus::Failing,
12621                        Some("health probe did not answer"),
12622                        1,
12623                    )
12624                    .await
12625                    .unwrap();
12626                    assert!(child.is_none());
12627                });
12628        });
12629        // The loopback barrier acknowledges entry into terminal recording by
12630        // health_restart_child's drain, then waits for the test's release byte.
12631        // Status and history are observed while that writer is held, not after
12632        // an elapsed-time guess about when the health action has completed.
12633        let mut reached = [0];
12634        barrier.read_exact(&mut reached).unwrap();
12635        assert_eq!(&reached, b"R");
12636        let before_status = module.status().unwrap();
12637        let before_history = module.terminal_history();
12638        barrier.write_all(b"X").unwrap();
12639        reap.join().unwrap();
12640        assert!(before_history.entries.is_empty());
12641        assert_ne!(
12642            before_status.state,
12643            ModuleState::Failed,
12644            "Failed was visible before its terminal record could be written"
12645        );
12646        assert!(
12647            before_status.process_alive,
12648            "process death was visible before its terminal record could be written"
12649        );
12650        let failed = module.status().unwrap();
12651        assert_eq!(failed.state, ModuleState::Failed);
12652        assert!(!failed.process_alive);
12653        let history = module.terminal_history();
12654        assert_eq!(history.entries.len(), 1);
12655        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12656        assert_eq!(
12657            history.entries[0].disposition_detail.as_deref(),
12658            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12659        );
12660    }
12661
12662    #[test]
12663    fn observed_exits_publish_process_death_after_terminal_history() {
12664        use std::io::{Read, Write};
12665        for (kind, enabled, shutdown, wait_error, expected_state) in [
12666            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12667            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12668            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12669            (
12670                ExitKind::DeliberateSeverance,
12671                true,
12672                false,
12673                false,
12674                ModuleState::Restarting,
12675            ),
12676            #[cfg(unix)]
12677            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12678            (ExitKind::Crash, true, false, true, ModuleState::Failed),
12679        ] {
12680            let supervisor = Supervisor::new_for_test(
12681                Arc::new(Registry::default()),
12682                RestartPolicy::new(1, Duration::ZERO),
12683            );
12684            let runtime = supervisor.runtime_config();
12685            #[cfg(unix)]
12686            if shutdown {
12687                runtime.child_roster.close();
12688            }
12689            #[cfg(not(unix))]
12690            let _ = shutdown;
12691            let spec = windowed_crash_spec("exit-publication-order");
12692            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12693                ModuleState::Running,
12694                enabled,
12695            )));
12696            lock_snapshot(&snapshot).unwrap().process_alive = true;
12697            let reaped_snapshot = snapshot.clone();
12698            let history = runtime.terminal_ring.clone();
12699            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12700            let reap = std::thread::spawn(move || {
12701                if wait_error {
12702                    super::fail_after_wait_error(
12703                        &spec.module_id,
12704                        &reaped_snapshot,
12705                        &runtime.terminal_ring,
12706                        &runtime.spawn_events,
12707                        "wait failed".into(),
12708                        &runtime.failure_context(true),
12709                    );
12710                } else {
12711                    tokio::runtime::Builder::new_current_thread()
12712                        .enable_all()
12713                        .build()
12714                        .unwrap()
12715                        .block_on(on_child_exit(
12716                            &spec,
12717                            runtime.restart_policy,
12718                            &supervisor.registry,
12719                            &reaped_snapshot,
12720                            &runtime.terminal_ring,
12721                            &runtime.spawn_events,
12722                            &runtime.child_roster,
12723                            ExitReport {
12724                                kind,
12725                                code: Some(0),
12726                                signal: None,
12727                                at_ms: 1,
12728                            },
12729                            true,
12730                        ));
12731                }
12732            });
12733            let mut reached = [0];
12734            barrier.read_exact(&mut reached).unwrap();
12735            assert_eq!(&reached, b"R");
12736            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12737            let before_history = history.lock().unwrap().snapshot();
12738            barrier.write_all(b"X").unwrap();
12739            reap.join().unwrap();
12740            assert!(
12741                before_alive,
12742                "{kind:?}: death was published before recording"
12743            );
12744            assert!(before_history.entries.is_empty());
12745            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12746            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12747            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12748        }
12749    }
12750
12751    #[tokio::test]
12752    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12753        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12754        let path = dir.join("terminals.jsonl");
12755        std::fs::create_dir(&path).unwrap();
12756        let supervisor = Supervisor::new_for_test(
12757            Arc::new(Registry::default()),
12758            RestartPolicy::new(0, Duration::ZERO),
12759        )
12760        .with_terminal_journal(path, "health-budget-journal-failure".into());
12761        let runtime = supervisor.runtime_config();
12762        let spec = ModuleSpec {
12763            program: fake_aft_stub_path(),
12764            protocol: ModuleProtocol::None,
12765            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12766            ..windowed_crash_spec("health-budget-journal-failure")
12767        };
12768        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12769        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12770        health_restart_child(
12771            &spec,
12772            &runtime,
12773            &supervisor.registry,
12774            &supervisor.process_liveness,
12775            &snapshot,
12776            &mut child,
12777            SupervisorHealthStatus::Failing,
12778            None,
12779            1,
12780        )
12781        .await
12782        .unwrap();
12783        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12784        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12785        let history = runtime
12786            .terminal_ring
12787            .lock()
12788            .unwrap()
12789            .durable_history(&spec.module_id);
12790        assert!(history.journal_write_failures > 0);
12791        assert_eq!(history.entries.len(), 1);
12792        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12793    }
12794
12795    #[test]
12796    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12797        let supervisor = Supervisor::new_for_test(
12798            Arc::new(Registry::default()),
12799            RestartPolicy::new(0, Duration::ZERO),
12800        );
12801        let runtime = supervisor.runtime_config();
12802        let spec = ModuleSpec {
12803            program: fake_aft_stub_path(),
12804            protocol: ModuleProtocol::None,
12805            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12806            ..windowed_crash_spec("health-budget-recording-panic")
12807        };
12808        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12809        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12810            tokio::runtime::Builder::new_current_thread()
12811                .enable_all()
12812                .build()
12813                .unwrap()
12814                .block_on(async {
12815                    let mut child =
12816                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12817                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12818                    health_restart_child(
12819                        &spec,
12820                        &runtime,
12821                        &supervisor.registry,
12822                        &supervisor.process_liveness,
12823                        &snapshot,
12824                        &mut child,
12825                        SupervisorHealthStatus::Failing,
12826                        None,
12827                        1,
12828                    )
12829                    .await
12830                    .unwrap();
12831                });
12832        }));
12833        let panic = result.expect_err("recording must still unwind");
12834        assert_eq!(
12835            panic.downcast_ref::<String>().map(String::as_str),
12836            Some("spawn event sequence exhausted")
12837        );
12838        let state = lock_snapshot(&snapshot).unwrap();
12839        assert_eq!(state.state, ModuleState::Failed);
12840        assert!(!state.process_alive);
12841        assert!(state.last_exit.is_some());
12842    }
12843
12844    #[tokio::test]
12845    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12846        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12847        let path = dir.join("terminals.jsonl");
12848        std::fs::create_dir(&path).unwrap();
12849        let supervisor = Supervisor::new_for_test(
12850            Arc::new(Registry::default()),
12851            RestartPolicy::new(0, Duration::ZERO),
12852        )
12853        .with_terminal_journal(path, "budget-journal-failure".into());
12854        let runtime = supervisor.runtime_config();
12855        let spec = windowed_crash_spec("budget-journal-failure");
12856        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12857        assert!(matches!(
12858            on_child_exit(
12859                &spec,
12860                runtime.restart_policy,
12861                &supervisor.registry,
12862                &snapshot,
12863                &runtime.terminal_ring,
12864                &runtime.spawn_events,
12865                &runtime.child_roster,
12866                crash_exit_report(1),
12867                true,
12868            )
12869            .await,
12870            NextAction::Stop { .. }
12871        ));
12872        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12873        let history = runtime
12874            .terminal_ring
12875            .lock()
12876            .unwrap()
12877            .durable_history(&spec.module_id);
12878        assert!(history.journal_write_failures > 0);
12879        assert_eq!(history.entries.len(), 1);
12880        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12881    }
12882
12883    #[test]
12884    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12885        let supervisor = Supervisor::new_for_test(
12886            Arc::new(Registry::default()),
12887            RestartPolicy::new(0, Duration::ZERO),
12888        );
12889        let runtime = supervisor.runtime_config();
12890        let spec = windowed_crash_spec("budget-recording-panic");
12891        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12892        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12893        // Exhausting the event sequence makes emit_exited panic before the
12894        // terminal write, exercising publication on the recording unwind.
12895        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12896        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12897            tokio::runtime::Builder::new_current_thread()
12898                .enable_all()
12899                .build()
12900                .unwrap()
12901                .block_on(on_child_exit(
12902                    &spec,
12903                    runtime.restart_policy,
12904                    &supervisor.registry,
12905                    &snapshot,
12906                    &runtime.terminal_ring,
12907                    &runtime.spawn_events,
12908                    &runtime.child_roster,
12909                    crash_exit_report(1),
12910                    true,
12911                ))
12912        }));
12913        let panic = result.err().expect("recording must still unwind");
12914        assert_eq!(
12915            panic.downcast_ref::<String>().map(String::as_str),
12916            Some("spawn event sequence exhausted")
12917        );
12918        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12919    }
12920
12921    /// A stop the supervisor itself requests still stops a protocol-none
12922    /// module, even though the child answers the SIGTERM with exit 0.
12923    #[cfg(unix)]
12924    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12925    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12926        for disable in [false, true] {
12927            let label = if disable {
12928                "none-requested-disable"
12929            } else {
12930                "none-requested-stop"
12931            };
12932            let dir = cortexkit_test_support::ScratchDir::new(label);
12933            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12934            let supervisor = Supervisor::new_for_test(
12935                Arc::new(Registry::default()),
12936                RestartPolicy::new(3, Duration::ZERO),
12937            );
12938            let module = supervisor.spawn(spec).unwrap();
12939            wait_for_file(&ready).await;
12940
12941            if disable {
12942                module.set_enabled(false).await.unwrap();
12943            } else {
12944                module.stop().await.unwrap();
12945            }
12946            assert!(
12947                marker.exists(),
12948                "{label}: the child must have left through its SIGTERM handler with exit 0"
12949            );
12950
12951            // Long enough for a zero-backoff respawn to have happened if the
12952            // exit had been treated as a crash.
12953            sleep(Duration::from_millis(500)).await;
12954            let status = module.status().unwrap();
12955            let expected = if disable {
12956                ModuleState::Disabled
12957            } else {
12958                ModuleState::Stopped
12959            };
12960            assert_eq!(status.state, expected, "{label}");
12961            assert_eq!(
12962                status.spawn_generation, 1,
12963                "{label}: respawned after a requested stop"
12964            );
12965            let history = module.terminal_history();
12966            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12967            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12968            assert_ne!(
12969                history.entries[0].disposition,
12970                TerminalDisposition::Restarting,
12971                "{label}"
12972            );
12973        }
12974    }
12975
12976    /// A subc-wire module that exits 0 on its own is still a stop: the
12977    /// protocol-none rule must not reach it.
12978    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12979    async fn subc_wire_clean_exit_is_still_a_stop() {
12980        let supervisor = Supervisor::new_for_test(
12981            Arc::new(Registry::default()),
12982            RestartPolicy::new(3, Duration::ZERO),
12983        );
12984        let module = supervisor
12985            .spawn(ModuleSpec {
12986                module_id: "wire-clean-exit".to_string(),
12987                program: fake_aft_stub_path(),
12988                args: Vec::new(),
12989                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12990                reserved: false,
12991                reserved_prefixes: Vec::new(),
12992                protocol: ModuleProtocol::Subc,
12993                overlap: Default::default(),
12994            })
12995            .unwrap();
12996
12997        let deadline = Instant::now() + Duration::from_secs(10);
12998        while module.terminal_history().entries.is_empty() {
12999            assert!(Instant::now() < deadline, "module never exited");
13000            sleep(Duration::from_millis(10)).await;
13001        }
13002        // Long enough for a zero-backoff respawn to have happened.
13003        sleep(Duration::from_millis(500)).await;
13004        let status = module.status().unwrap();
13005        assert_eq!(status.state, ModuleState::Stopped);
13006        assert_eq!(status.spawn_generation, 1);
13007        let history = module.terminal_history();
13008        assert_eq!(history.entries.len(), 1, "{history:?}");
13009        assert_eq!(history.entries[0].exit_code, Some(0));
13010        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13011    }
13012
13013    #[cfg(unix)]
13014    #[tokio::test]
13015    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13016        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13017        let record = dir.join("live-children.json");
13018        let supervisor = Supervisor::new_for_test(
13019            Arc::new(Registry::default()),
13020            RestartPolicy::new(0, Duration::ZERO),
13021        );
13022        let mut runtime = supervisor.runtime_config();
13023        runtime.child_roster.record_to(record.clone());
13024        let gate = Arc::new(super::ReloadExitRecordGate::default());
13025        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13026        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13027        let spec = ModuleSpec {
13028            module_id: "reload-exit-roster".into(),
13029            program: fake_aft_stub_path(),
13030            args: Vec::new(),
13031            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13032            reserved: false,
13033            reserved_prefixes: Vec::new(),
13034            protocol: ModuleProtocol::Subc,
13035            overlap: Default::default(),
13036        };
13037        let mut child = None;
13038        let reload = super::finish_reload_child(
13039            &spec,
13040            &runtime,
13041            &supervisor.registry,
13042            &supervisor.process_liveness,
13043            &snapshot,
13044            &mut child,
13045        );
13046        tokio::pin!(reload);
13047        tokio::select! {
13048            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13049            _ = gate.reached.notified() => {}
13050        }
13051        assert!(runtime
13052            .terminal_ring
13053            .lock()
13054            .unwrap()
13055            .snapshot()
13056            .entries
13057            .is_empty());
13058        assert_eq!(
13059            crate::live_children::read_record(&record).unwrap().len(),
13060            1,
13061            "shutdown must still wait for the reaped child until its terminal record exists"
13062        );
13063        runtime.child_roster.close();
13064        gate.resume.notify_one();
13065        assert!(reload.await.is_err());
13066        assert!(crate::live_children::read_record(&record)
13067            .unwrap()
13068            .is_empty());
13069        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13070        assert_eq!(history.entries.len(), 1);
13071        assert_eq!(
13072            history.entries[0].disposition,
13073            TerminalDisposition::DaemonShutdown
13074        );
13075    }
13076
13077    /// Each restart-producing arm has its own state transition. Keeping their
13078    /// lifetime count assertions adjacent prevents a later new arm from silently
13079    /// spending budget without recording the historical restart.
13080    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13081    async fn every_restart_increment_path_advances_lifetime_count() {
13082        let supervisor = Supervisor::new_for_test(
13083            Arc::new(Registry::default()),
13084            RestartPolicy::new(1, Duration::ZERO),
13085        );
13086        let runtime = supervisor.runtime_config();
13087        let spec = ModuleSpec {
13088            module_id: "lifetime-increment-path".to_string(),
13089            program: PathBuf::from("/unused/lifetime-increment-path"),
13090            args: Vec::new(),
13091            env: Vec::new(),
13092            reserved: false,
13093            reserved_prefixes: Vec::new(),
13094            protocol: ModuleProtocol::Subc,
13095            overlap: Default::default(),
13096        };
13097
13098        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13099        assert!(matches!(
13100            on_child_exit(
13101                &spec,
13102                runtime.restart_policy,
13103                &supervisor.registry,
13104                &crash_snapshot,
13105                &runtime.terminal_ring,
13106                &runtime.spawn_events,
13107                &runtime.child_roster,
13108                ExitReport {
13109                    kind: ExitKind::Crash,
13110                    code: Some(1),
13111                    signal: None,
13112                    at_ms: 1,
13113                },
13114                true,
13115            )
13116            .await,
13117            NextAction::Restart { schedule: _ }
13118        ));
13119        let (crash_restarts, crash_lifetime) = {
13120            let state = lock_snapshot(&crash_snapshot).unwrap();
13121            (state.crash_restarts.len(), state.lifetime_restarts)
13122        };
13123        assert_eq!(crash_restarts, 1);
13124        assert_eq!(crash_lifetime, 1);
13125
13126        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13127        let mut health_child = None;
13128        assert!(matches!(
13129            health_restart_child(
13130                &spec,
13131                &runtime,
13132                &supervisor.registry,
13133                &supervisor.process_liveness,
13134                &health_snapshot,
13135                &mut health_child,
13136                SupervisorHealthStatus::Failing,
13137                None,
13138                2,
13139            )
13140            .await,
13141            Ok(())
13142        ));
13143        assert!(health_child.is_none());
13144        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
13145        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
13146        let (health_restarts, health_lifetime) = {
13147            let state = lock_snapshot(&health_snapshot).unwrap();
13148            (state.crash_restarts.len(), state.lifetime_restarts)
13149        };
13150        assert_eq!(health_restarts, 1);
13151        assert_eq!(health_lifetime, 1);
13152
13153        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13154        let mut reload_child = None;
13155        assert!(matches!(
13156            handle_reload_spawn_failure(
13157                &spec,
13158                &runtime,
13159                &supervisor.process_liveness,
13160                &reload_snapshot,
13161                &mut reload_child,
13162                "forced reload spawn failure".to_string(),
13163            )
13164            .await,
13165            Err(SuperviseError::ReloadFailed { .. })
13166        ));
13167        let (reload_restarts, reload_lifetime) = {
13168            let state = lock_snapshot(&reload_snapshot).unwrap();
13169            (state.crash_restarts.len(), state.lifetime_restarts)
13170        };
13171        assert_eq!(reload_restarts, 1);
13172        assert_eq!(reload_lifetime, 1);
13173    }
13174
13175    #[tokio::test]
13176    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
13177        let supervisor = Supervisor::new_for_test(
13178            Arc::new(Registry::default()),
13179            RestartPolicy::new(3, Duration::ZERO),
13180        );
13181        let runtime = supervisor.runtime_config();
13182        let spec = ModuleSpec {
13183            module_id: "deliberately-severed".to_string(),
13184            program: PathBuf::from("/unused/deliberately-severed"),
13185            args: Vec::new(),
13186            env: Vec::new(),
13187            reserved: false,
13188            reserved_prefixes: Vec::new(),
13189            protocol: ModuleProtocol::Subc,
13190            overlap: Default::default(),
13191        };
13192        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13193        let process = ProcessIdentity {
13194            pid: 41,
13195            start_time: 101,
13196        };
13197        record_deliberate_severance(&snapshot, process).unwrap();
13198        let exit_report = apply_deliberate_severance_marker(
13199            &snapshot,
13200            Some(process),
13201            ExitReport {
13202                kind: ExitKind::Crash,
13203                code: Some(1),
13204                signal: None,
13205                at_ms: 1,
13206            },
13207        );
13208        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
13209
13210        assert!(matches!(
13211            on_child_exit(
13212                &spec,
13213                runtime.restart_policy,
13214                &supervisor.registry,
13215                &snapshot,
13216                &runtime.terminal_ring,
13217                &runtime.spawn_events,
13218                &runtime.child_roster,
13219                exit_report,
13220                true,
13221            )
13222            .await,
13223            NextAction::Restart { schedule: _ }
13224        ));
13225        let state = lock_snapshot(&snapshot).unwrap();
13226        assert_eq!(state.lifetime_restarts, 1);
13227        assert_eq!(state.crash_restarts.len(), 0);
13228    }
13229
13230    #[tokio::test]
13231    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
13232        let supervisor = Supervisor::new_for_test(
13233            Arc::new(Registry::default()),
13234            RestartPolicy::new(3, Duration::ZERO),
13235        );
13236        let runtime = supervisor.runtime_config();
13237        let spec = ModuleSpec {
13238            module_id: "genuine-crash".to_string(),
13239            program: PathBuf::from("/unused/genuine-crash"),
13240            args: Vec::new(),
13241            env: Vec::new(),
13242            reserved: false,
13243            reserved_prefixes: Vec::new(),
13244            protocol: ModuleProtocol::Subc,
13245            overlap: Default::default(),
13246        };
13247        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13248
13249        assert!(matches!(
13250            on_child_exit(
13251                &spec,
13252                runtime.restart_policy,
13253                &supervisor.registry,
13254                &snapshot,
13255                &runtime.terminal_ring,
13256                &runtime.spawn_events,
13257                &runtime.child_roster,
13258                ExitReport {
13259                    kind: ExitKind::Crash,
13260                    code: Some(1),
13261                    signal: None,
13262                    at_ms: 1,
13263                },
13264                true,
13265            )
13266            .await,
13267            NextAction::Restart { schedule: _ }
13268        ));
13269        let state = lock_snapshot(&snapshot).unwrap();
13270        assert_eq!(state.lifetime_restarts, 1);
13271        assert_eq!(state.crash_restarts.len(), 1);
13272    }
13273
13274    fn crash_exit_report(at_ms: u64) -> ExitReport {
13275        ExitReport {
13276            kind: ExitKind::Crash,
13277            code: Some(1),
13278            signal: None,
13279            at_ms,
13280        }
13281    }
13282
13283    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
13284        ModuleSpec {
13285            module_id: module_id.to_string(),
13286            program: PathBuf::from("/unused").join(module_id),
13287            args: Vec::new(),
13288            env: Vec::new(),
13289            reserved: false,
13290            reserved_prefixes: Vec::new(),
13291            protocol: ModuleProtocol::Subc,
13292            overlap: Default::default(),
13293        }
13294    }
13295
13296    /// A real crash loop still stops. Three crashes with nothing aging out spend
13297    /// a budget of two and the third respawn is refused, and both surfaces an
13298    /// operator has -- the log line and the retained terminal record -- name the
13299    /// window rather than only the cap, because `max_restarts=2` alone is what
13300    /// this budget used to mean.
13301    #[tokio::test]
13302    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
13303        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
13304        let supervisor = Supervisor::new_for_test(
13305            Arc::new(Registry::default()),
13306            RestartPolicy::new(2, Duration::ZERO),
13307        );
13308        let runtime = supervisor.runtime_config();
13309        let spec = windowed_crash_spec("crash-loop-in-window");
13310        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13311
13312        for attempt in 1..=2 {
13313            assert!(
13314                matches!(
13315                    on_child_exit(
13316                        &spec,
13317                        runtime.restart_policy,
13318                        &supervisor.registry,
13319                        &snapshot,
13320                        &runtime.terminal_ring,
13321                        &runtime.spawn_events,
13322                        &runtime.child_roster,
13323                        crash_exit_report(attempt),
13324                        true,
13325                    )
13326                    .await,
13327                    NextAction::Restart { schedule: _ }
13328                ),
13329                "crash {attempt} is inside the budget and must respawn"
13330            );
13331        }
13332
13333        assert!(matches!(
13334            on_child_exit(
13335                &spec,
13336                runtime.restart_policy,
13337                &supervisor.registry,
13338                &snapshot,
13339                &runtime.terminal_ring,
13340                &runtime.spawn_events,
13341                &runtime.child_roster,
13342                crash_exit_report(3),
13343                true,
13344            )
13345            .await,
13346            NextAction::Stop { .. }
13347        ));
13348
13349        {
13350            let state = lock_snapshot(&snapshot).unwrap();
13351            assert_eq!(state.state, ModuleState::Failed);
13352            assert_eq!(state.crash_restarts.len(), 2);
13353            assert_eq!(state.lifetime_restarts, 2);
13354        }
13355
13356        let history = runtime
13357            .terminal_ring
13358            .lock()
13359            .expect("terminal ring is not poisoned")
13360            .snapshot();
13361        let last = history
13362            .entries
13363            .last()
13364            .expect("the refused crash is retained");
13365        assert_eq!(last.disposition, TerminalDisposition::Failed);
13366        assert_eq!(
13367            last.disposition_detail.as_deref(),
13368            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
13369        );
13370
13371        let captured = crate::router::test_log::captured_logs(&logs);
13372        assert!(
13373            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
13374            "the stop must be logged with its window: {captured}"
13375        );
13376    }
13377
13378    /// The rate, stated as a test: three crashes where the first has aged past
13379    /// the window are two crashes as far as the budget is concerned, so the
13380    /// third respawn is allowed and the ring holds only the two recent ones.
13381    ///
13382    /// This is the case a lifetime counter got wrong -- and the case the daemon
13383    /// now hits routinely, since a module exits non-zero every time its
13384    /// connection to the daemon drops.
13385    #[tokio::test]
13386    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
13387        let supervisor = Supervisor::new_for_test(
13388            Arc::new(Registry::default()),
13389            RestartPolicy::new(2, Duration::ZERO),
13390        );
13391        let runtime = supervisor.runtime_config();
13392        let spec = windowed_crash_spec("crash-across-windows");
13393        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13394
13395        for attempt in 1..=2 {
13396            assert!(matches!(
13397                on_child_exit(
13398                    &spec,
13399                    runtime.restart_policy,
13400                    &supervisor.registry,
13401                    &snapshot,
13402                    &runtime.terminal_ring,
13403                    &runtime.spawn_events,
13404                    &runtime.child_roster,
13405                    crash_exit_report(attempt),
13406                    true,
13407                )
13408                .await,
13409                NextAction::Restart { schedule: _ }
13410            ));
13411        }
13412
13413        // The oldest crash moves out of the window; nothing else about the
13414        // module changes.
13415        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13416            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
13417        })
13418        .unwrap();
13419
13420        assert!(
13421            matches!(
13422                on_child_exit(
13423                    &spec,
13424                    runtime.restart_policy,
13425                    &supervisor.registry,
13426                    &snapshot,
13427                    &runtime.terminal_ring,
13428                    &runtime.spawn_events,
13429                    &runtime.child_roster,
13430                    crash_exit_report(3),
13431                    true,
13432                )
13433                .await,
13434                NextAction::Restart { schedule: _ }
13435            ),
13436            "a crash older than the window must not hold a budget slot"
13437        );
13438
13439        let state = lock_snapshot(&snapshot).unwrap();
13440        assert_eq!(state.state, ModuleState::Restarting);
13441        assert_eq!(
13442            state.crash_restarts.len(),
13443            2,
13444            "the aged instant is dropped and the new one takes its place"
13445        );
13446        assert_eq!(
13447            state.lifetime_restarts, 3,
13448            "the ledger counts every restart, including the ones the window forgot"
13449        );
13450    }
13451
13452    /// An operator restart hands the budget back whole, and the ledger keeps
13453    /// counting. Those are different questions -- "how close is this module to
13454    /// being stopped" and "how many times has it been replaced" -- and the
13455    /// operator action answers only the first.
13456    #[tokio::test]
13457    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13458        let supervisor = Supervisor::new_for_test(
13459            Arc::new(Registry::default()),
13460            RestartPolicy::new(2, Duration::ZERO),
13461        );
13462        let runtime = supervisor.runtime_config();
13463        let spec = windowed_crash_spec("operator-cleared-budget");
13464        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13465
13466        for attempt in 1..=2 {
13467            assert!(matches!(
13468                on_child_exit(
13469                    &spec,
13470                    runtime.restart_policy,
13471                    &supervisor.registry,
13472                    &snapshot,
13473                    &runtime.terminal_ring,
13474                    &runtime.spawn_events,
13475                    &runtime.child_roster,
13476                    crash_exit_report(attempt),
13477                    true,
13478                )
13479                .await,
13480                NextAction::Restart { schedule: _ }
13481            ));
13482        }
13483
13484        reset_restart_count(&snapshot, &spec.module_id).unwrap();
13485        {
13486            let state = lock_snapshot(&snapshot).unwrap();
13487            assert!(
13488                state.crash_restarts.is_empty(),
13489                "an operator restart returns the full budget"
13490            );
13491            assert_eq!(
13492                state.lifetime_restarts, 2,
13493                "clearing the budget must not unmake the crashes"
13494            );
13495        }
13496
13497        assert!(
13498            matches!(
13499                on_child_exit(
13500                    &spec,
13501                    runtime.restart_policy,
13502                    &supervisor.registry,
13503                    &snapshot,
13504                    &runtime.terminal_ring,
13505                    &runtime.spawn_events,
13506                    &runtime.child_roster,
13507                    crash_exit_report(3),
13508                    true,
13509                )
13510                .await,
13511                NextAction::Restart { schedule: _ }
13512            ),
13513            "the cleared budget must be spendable again"
13514        );
13515        let state = lock_snapshot(&snapshot).unwrap();
13516        assert_eq!(state.crash_restarts.len(), 1);
13517        assert_eq!(state.lifetime_restarts, 3);
13518    }
13519
13520    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13521    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13522        let severed = ProcessIdentity {
13523            pid: 41,
13524            start_time: 101,
13525        };
13526        let successor = ProcessIdentity {
13527            pid: 41,
13528            start_time: 202,
13529        };
13530        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13531        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13532            state.pid = Some(successor.pid);
13533            state.process_start_time = Some(successor.start_time);
13534        })
13535        .unwrap();
13536        assert!(!module.record_deliberate_severance(severed).unwrap());
13537
13538        let exit_report = apply_deliberate_severance_marker(
13539            &module.inner.snapshot,
13540            Some(successor),
13541            ExitReport {
13542                kind: ExitKind::Crash,
13543                code: Some(1),
13544                signal: None,
13545                at_ms: 1,
13546            },
13547        );
13548
13549        assert_eq!(exit_report.kind, ExitKind::Crash);
13550    }
13551
13552    #[tokio::test]
13553    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13554        let registry = Registry::default();
13555        let supervisor = Supervisor::new_for_test(
13556            Arc::new(Registry::default()),
13557            RestartPolicy::new(3, Duration::ZERO),
13558        );
13559        let runtime = supervisor.runtime_config();
13560        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13561        let spec = ModuleSpec {
13562            module_id: "drain-deliberate-severance".to_string(),
13563            program: fake_aft_stub_path(),
13564            args: Vec::new(),
13565            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13566            reserved: false,
13567            reserved_prefixes: Vec::new(),
13568            protocol: ModuleProtocol::Subc,
13569            overlap: Default::default(),
13570        };
13571        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13572        let process = ProcessIdentity {
13573            pid: 41,
13574            start_time: 101,
13575        };
13576        child.process_identity = Some(process);
13577        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13578            state.pid = Some(process.pid);
13579            state.process_start_time = Some(process.start_time);
13580        })
13581        .unwrap();
13582        record_deliberate_severance(&snapshot, process).unwrap();
13583
13584        drain_child_to_state(
13585            &spec.module_id,
13586            spec.protocol,
13587            // The child exits on its own; no signal may change the exit this
13588            // test classifies.
13589            StopNotice::SentOverConnection,
13590            &registry,
13591            None,
13592            &snapshot,
13593            &runtime.terminal_ring,
13594            &runtime.spawn_events,
13595            child,
13596            Duration::from_secs(1),
13597            ModuleState::Stopped,
13598            Some(false),
13599            &super::FailureContext::operator(),
13600        )
13601        .await
13602        .unwrap();
13603
13604        let state = lock_snapshot(&snapshot).unwrap();
13605        assert_eq!(
13606            state.last_exit.as_ref().map(|exit| exit.kind),
13607            Some(ExitKind::DeliberateSeverance)
13608        );
13609        assert_eq!(state.lifetime_restarts, 1);
13610        assert_eq!(state.crash_restarts.len(), 0);
13611        drop(state);
13612        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13613        assert_eq!(
13614            history.entries[0].exit_kind,
13615            subc_control::TerminalExitKind::DeliberateSeverance
13616        );
13617    }
13618
13619    #[tokio::test]
13620    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13621        let registry = Registry::default();
13622        let supervisor = Supervisor::new_for_test(
13623            Arc::new(Registry::default()),
13624            RestartPolicy::new(3, Duration::ZERO),
13625        );
13626        let runtime = supervisor.runtime_config();
13627        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13628        let spec = ModuleSpec {
13629            module_id: "ordinary-drain".to_string(),
13630            program: fake_aft_stub_path(),
13631            args: Vec::new(),
13632            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13633            reserved: false,
13634            reserved_prefixes: Vec::new(),
13635            protocol: ModuleProtocol::Subc,
13636            overlap: Default::default(),
13637        };
13638        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13639
13640        drain_child_to_state(
13641            &spec.module_id,
13642            spec.protocol,
13643            // The child exits on its own; no signal may change the exit this
13644            // test classifies.
13645            StopNotice::SentOverConnection,
13646            &registry,
13647            None,
13648            &snapshot,
13649            &runtime.terminal_ring,
13650            &runtime.spawn_events,
13651            child,
13652            Duration::from_secs(1),
13653            ModuleState::Stopped,
13654            Some(false),
13655            &super::FailureContext::operator(),
13656        )
13657        .await
13658        .unwrap();
13659
13660        let state = lock_snapshot(&snapshot).unwrap();
13661        assert_eq!(
13662            state.last_exit.as_ref().map(|exit| exit.kind),
13663            Some(ExitKind::Crash)
13664        );
13665        assert_eq!(state.lifetime_restarts, 0);
13666        assert_eq!(state.crash_restarts.len(), 0);
13667    }
13668
13669    #[test]
13670    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13671        // The server's generic fatal-routing branch only knows that the
13672        // connection failed; it does not know that the daemon deliberately
13673        // initiated a process-killing severance. Keep this seam explicit so a
13674        // future connection error path cannot silently reintroduce the stale
13675        // exemption that mislabels a later genuine crash.
13676        assert!(!include_str!("server.rs")
13677            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13678    }
13679
13680    /// The `route.closed` `drained` value must be the quiescence wait's own
13681    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
13682    /// measurement at all and `false` is the one honest constant. This is the exact
13683    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
13684    /// on every return path, including the one that used to return early via `?`
13685    /// with `route.closing` already sent and no `route.closed` ever following.
13686    #[test]
13687    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13688        assert!(drained_after_quiescence_wait(&Ok(true)));
13689        assert!(!drained_after_quiescence_wait(&Ok(false)));
13690        assert!(!drained_after_quiescence_wait(&Err(
13691            SuperviseError::StatePoisoned { module_id: None }
13692        )));
13693    }
13694
13695    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
13696    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
13697    /// already reaped out-of-band) still leaves a terminal record rather than none
13698    /// at all. Triggering the real `wait()` I/O error from an integration test would
13699    /// need a genuine already-reaped-child race, which is OS-specific and not
13700    /// something this suite attempts elsewhere; this test instead verifies the
13701    /// record produced for that arm end-to-end through the real `TerminalRing`, and
13702    /// the call site itself is verified by inspection to sit in that exact arm.
13703    #[test]
13704    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13705        let ring = Arc::new(Mutex::new(TerminalRing::new(
13706            TerminalRingConfig::default(),
13707            0,
13708        )));
13709        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13710
13711        let snapshot = ring.lock().unwrap().snapshot();
13712        assert_eq!(snapshot.entries.len(), 1);
13713        let entry = &snapshot.entries[0];
13714        assert_eq!(entry.exit_code, None);
13715        assert_eq!(entry.exit_signal, None);
13716        assert_eq!(entry.disposition, TerminalDisposition::Failed);
13717    }
13718
13719    #[test]
13720    fn wait_error_exit_path_preserves_spawn_event_density() {
13721        let feed = super::SpawnEventFeed::default();
13722        feed.configure_incarnation("wait-error-density".to_string());
13723        feed.emit_spawned("wait-error", 41, 1);
13724        let ring = Arc::new(Mutex::new(TerminalRing::new(
13725            TerminalRingConfig::default(),
13726            0,
13727        )));
13728
13729        record_wait_error_terminal("wait-error", &ring, &feed);
13730        feed.emit_spawned("after-wait-error", 42, 2);
13731
13732        let state = feed.0.lock().unwrap();
13733        let sequences = state
13734            .events
13735            .iter()
13736            .map(|event| event.cursor.seq)
13737            .collect::<Vec<_>>();
13738        assert_eq!(sequences, vec![1, 2, 3]);
13739        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13740        assert_eq!(state.events[1].exit_code, None);
13741        assert_eq!(state.events[1].exit_signal, None);
13742    }
13743
13744    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
13745    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
13746    /// not a clean exit it never actually observed.
13747    #[test]
13748    fn wait_error_exit_report_is_classified_as_a_crash() {
13749        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13750    }
13751}
13752
13753#[cfg(all(test, unix))]
13754mod restart_ordering_tests {
13755    use super::*;
13756    use std::io::Write;
13757    use tokio::io::AsyncReadExt;
13758
13759    #[test]
13760    #[ignore = "child process fixture"]
13761    fn serving_child() {
13762        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13763            return;
13764        };
13765        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13766        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13767        for stream in listener.incoming() {
13768            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13769        }
13770    }
13771
13772    async fn restart_before_publication(requests: usize) {
13773        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13774        let address_file = home.path().join("address");
13775        let spec = ModuleSpec {
13776            module_id: "restart-serving-child".into(),
13777            program: std::env::current_exe().unwrap(),
13778            args: vec![
13779                "--exact".into(),
13780                "supervise::restart_ordering_tests::serving_child".into(),
13781                "--ignored".into(),
13782                "--nocapture".into(),
13783            ],
13784            env: vec![(
13785                "RESTART_TEST_ADDRESS_FILE".into(),
13786                address_file.display().to_string(),
13787            )],
13788            reserved: false,
13789            reserved_prefixes: vec![],
13790            protocol: ModuleProtocol::None,
13791            overlap: Default::default(),
13792        };
13793        let mut supervisor =
13794            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13795        let (spawned_tx, spawned_rx) = oneshot::channel();
13796        let spawned_tx = Mutex::new(Some(spawned_tx));
13797        let (release_tx, release_rx) = std::sync::mpsc::channel();
13798        let release_rx = Mutex::new(release_rx);
13799        // Hold publication, not the child: it must answer a real request before
13800        // the restart enters the queue, even though status still has generation 0.
13801        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13802            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13803            release_rx
13804                .lock()
13805                .unwrap()
13806                .recv_timeout(Duration::from_secs(20))
13807                .unwrap();
13808        })));
13809        let registered = supervisor
13810            .register_configured_with_health(
13811                spec.clone(),
13812                true,
13813                HealthConfig::default(),
13814                None,
13815                RestartPolicy::new(3, Duration::ZERO),
13816            )
13817            .unwrap();
13818        let module = registered.module.clone();
13819        let startup = tokio::task::spawn_blocking(move || {
13820            supervisor.start_registered_configured(registered, None, true)
13821        });
13822        spawned_rx.await.unwrap();
13823        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13824            let address = loop {
13825                if let Ok(address) = std::fs::read_to_string(&address_file) {
13826                    if !address.is_empty() {
13827                        break address;
13828                    }
13829                }
13830                tokio::time::sleep(Duration::from_millis(1)).await;
13831            };
13832            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13833            let mut reply = String::new();
13834            stream.read_to_string(&mut reply).await.unwrap();
13835            reply.trim().parse::<u32>().unwrap()
13836        })
13837        .await
13838        .expect("child must serve before generation publication");
13839        assert_eq!(
13840            lock_snapshot(&module.inner.snapshot)
13841                .unwrap()
13842                .spawn_generation,
13843            0
13844        );
13845        let mut replies = Vec::new();
13846        for _ in 0..requests {
13847            let mut restart = Box::pin(module.restart(Some(0)));
13848            // Poll the public API through queue admission, leaving only its ACK
13849            // pending. The monitor cannot consume the command until released.
13850            std::future::poll_fn(|cx| {
13851                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13852                std::task::Poll::Ready(())
13853            })
13854            .await;
13855            replies.push(restart);
13856        }
13857        release_tx.send(()).unwrap();
13858        startup.await.unwrap().unwrap();
13859        for reply in replies {
13860            reply.await.unwrap();
13861        }
13862        // Restart schedules its replacement after completing the handler. Wait
13863        // for that separate actor turn before placing the command barrier.
13864        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
13865            loop {
13866                if lock_snapshot(&module.inner.snapshot)
13867                    .unwrap()
13868                    .spawn_generation
13869                    > 1
13870                {
13871                    break;
13872                }
13873                tokio::time::sleep(Duration::from_millis(1)).await;
13874            }
13875        })
13876        .await;
13877        // Commands are handled serially. The update-spec reply proves every
13878        // restart handler has finished before we inspect the snapshot; restart's
13879        // own ACK only acknowledges initiation, not replacement completion.
13880        module.update_spec_for_test(spec).await.unwrap();
13881        let (generation, pid) = {
13882            let state = lock_snapshot(&module.inner.snapshot).unwrap();
13883            (state.spawn_generation, state.pid)
13884        };
13885        module.stop().await.unwrap();
13886        replacement.expect("restart must publish a replacement for the already-serving child");
13887        assert_ne!(
13888            pid,
13889            Some(old_pid),
13890            "a serving child predating the restart must be replaced"
13891        );
13892        assert_eq!(generation, 2, "queued restarts must share one replacement");
13893    }
13894
13895    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13896    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
13897        restart_before_publication(1).await;
13898    }
13899
13900    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13901    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
13902        restart_before_publication(2).await;
13903    }
13904
13905    #[tokio::test]
13906    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
13907        let supervisor =
13908            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13909        let registered = supervisor
13910            .register_configured_with_health(
13911                ModuleSpec {
13912                    module_id: "queued-before-replacement".into(),
13913                    // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on
13914                    // macOS a module whose running image is not the configured
13915                    // program is refused once its exec is confirmed (see
13916                    // `record_write_stall_tests::sleeping_spec`).
13917                    program: "/bin/sleep".into(),
13918                    args: vec!["600".into()],
13919                    env: vec![],
13920                    reserved: false,
13921                    reserved_prefixes: vec![],
13922                    protocol: ModuleProtocol::None,
13923                    overlap: Default::default(),
13924                },
13925                true,
13926                HealthConfig::default(),
13927                None,
13928                RestartPolicy::default(),
13929            )
13930            .unwrap();
13931        let RegisteredConfiguredModule {
13932            mut spec,
13933            mut runtime,
13934            module,
13935            mut commands,
13936        } = registered;
13937        let snapshot = &module.inner.snapshot;
13938        let mut restart = Box::pin(module.restart(Some(0)));
13939        std::future::poll_fn(|cx| {
13940            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13941            std::task::Poll::Ready(())
13942        })
13943        .await;
13944        // Drive the actor explicitly so a replacement is definitely spawned
13945        // after receipt, but before this queued command is dequeued.
13946        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
13947        let replacement_pid = child.as_ref().unwrap().id();
13948        let command = commands.recv().await.unwrap();
13949        assert!(
13950            handle_supervisor_command(
13951                command,
13952                &mut spec,
13953                &mut runtime,
13954                &supervisor.registry,
13955                &supervisor.process_liveness,
13956                snapshot,
13957                &mut child,
13958                &mut commands,
13959                &mut VecDeque::new(),
13960            )
13961            .await
13962        );
13963        restart.await.unwrap();
13964        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
13965        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
13966        if let Some(mut child) = child {
13967            child.start_kill().unwrap();
13968            child.wait().await.unwrap();
13969        }
13970        assert_eq!(
13971            kept_pid, replacement_pid,
13972            "a newer replacement must survive"
13973        );
13974        assert!(
13975            !pending,
13976            "a satisfied restart must not schedule another spawn"
13977        );
13978    }
13979}
13980
13981#[cfg(all(test, unix))]
13982mod health_event_tests {
13983    use super::*;
13984    use cortexkit_test_support::ScratchDir;
13985    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13986
13987    struct Actor {
13988        module: SupervisedModule,
13989        registry: Arc<Registry>,
13990        forwarding: Arc<ForwardingTable>,
13991        spec: ModuleSpec,
13992        health: HealthConfig,
13993        rx: mpsc::Receiver<crate::router::OutboundFrame>,
13994        _home: ScratchDir,
13995    }
13996
13997    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13998        // OS exec and socket readiness are real I/O. Stay runnable while waiting
13999        // for an observed condition, rather than letting paused time auto-advance
14000        // deadlines or assuming a fixed number of yields finishes the work.
14001        let deadline = std::time::Instant::now() + Duration::from_secs(10);
14002        let tick = Instant::now();
14003        loop {
14004            if let Some(value) = observe() {
14005                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14006                return value;
14007            }
14008            assert!(
14009                std::time::Instant::now() < deadline,
14010                "timed out waiting for {reason}"
14011            );
14012            tokio::task::yield_now().await;
14013        }
14014    }
14015
14016    impl Actor {
14017        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14018            let home = ScratchDir::new("health-events");
14019            let registry = Arc::new(Registry::default());
14020            let forwarding = Arc::new(ForwardingTable::default());
14021            let health = HealthConfig {
14022                cadence,
14023                deadline: Duration::from_secs(3600),
14024                ..HealthConfig::default()
14025            };
14026            let supervisor =
14027                Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
14028                    .with_forwarding(forwarding.clone())
14029                    .with_health_config(health.clone());
14030            let spec = ModuleSpec {
14031                module_id: "event-health-child".into(),
14032                // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on macOS
14033                // the privacy trampoline refuses a module whose running image
14034                // is not the configured program, and `start` below waits for
14035                // the process to be confirmed (`reported_pid`). The shell form
14036                // passed only while the image check ran before the shell had
14037                // replaced itself with `sleep` (see
14038                // `record_write_stall_tests::sleeping_spec`).
14039                program: PathBuf::from("/bin/sleep"),
14040                // The process stays alive but has no real bus peer. Registrations
14041                // below exercise the same registry writes as accepted HELLOs.
14042                args: vec!["600".into()],
14043                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14044                    .into_iter()
14045                    .map(|key| (key.into(), home.path().display().to_string()))
14046                    .collect(),
14047                reserved: false,
14048                reserved_prefixes: vec![],
14049                protocol,
14050                overlap: Default::default(),
14051            };
14052            let module = supervisor.spawn(spec.clone()).unwrap();
14053            let (_, rx) = mpsc::channel(8);
14054            let actor = Self {
14055                module,
14056                registry,
14057                forwarding,
14058                spec,
14059                health,
14060                rx,
14061                _home: home,
14062            };
14063            actor
14064                .wait_for_select(|checkpoint| checkpoint.generation > 0)
14065                .await;
14066            actor
14067        }
14068
14069        fn turns(&self) -> u64 {
14070            lock_snapshot(&self.module.inner.snapshot)
14071                .unwrap()
14072                .actor_turns
14073        }
14074
14075        async fn wait_for_select(
14076            &self,
14077            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14078        ) -> ActorSelectCheckpoint {
14079            wait_for("actor select checkpoint", || {
14080                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14081                state.actor_select.filter(|checkpoint| {
14082                    checkpoint.generation == state.spawn_generation
14083                        && state.state == ModuleState::Running
14084                        && state.process_alive
14085                        && state.reported_pid().is_some()
14086                        && expected(checkpoint)
14087                })
14088            })
14089            .await
14090        }
14091
14092        fn hello(&mut self, connection: u64, health: bool) {
14093            let manifest =
14094                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
14095                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
14096                    .build();
14097            let connection = ConnectionId::new(connection);
14098            let (tx, rx) = mpsc::channel(8);
14099            self.rx = rx;
14100            self.forwarding
14101                .register_module_connection(
14102                    connection,
14103                    self.spec.module_id.clone(),
14104                    subc_protocol::PROTOCOL_VERSION,
14105                    Concurrency::ModuleManaged,
14106                    FrameSink::new(tx),
14107                )
14108                .unwrap();
14109            self.registry
14110                .register_with_control_ops(
14111                    manifest,
14112                    subc_protocol::PROTOCOL_VERSION,
14113                    connection,
14114                    if health {
14115                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
14116                    } else {
14117                        vec![]
14118                    },
14119                )
14120                .unwrap();
14121        }
14122
14123        async fn probe(&mut self) -> crate::router::OutboundFrame {
14124            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
14125                Ok(frame) => Some(frame),
14126                Err(mpsc::error::TryRecvError::Empty) => None,
14127                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
14128            })
14129            .await;
14130            let body: Value = serde_json::from_slice(&frame.body).unwrap();
14131            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14132            frame
14133        }
14134
14135        fn no_probe(&mut self) {
14136            assert!(matches!(
14137                self.rx.try_recv(),
14138                Err(mpsc::error::TryRecvError::Empty)
14139            ));
14140        }
14141
14142        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
14143            self.forwarding
14144                .complete_module_control_rpc(
14145                    ConnectionId::new(connection),
14146                    frame.header.corr,
14147                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14148                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14149                        status: HealthStatus::Ok,
14150                        detail: None,
14151                        metrics: None,
14152                    }),
14153                )
14154                .unwrap();
14155        }
14156    }
14157
14158    #[tokio::test(start_paused = true)]
14159    async fn no_health_children_park_for_sixty_seconds() {
14160        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14161        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14162        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14163        unadvertised.hello(1, false);
14164        // Include any one-off registration wake in the budget; receiving a
14165        // no-health HELLO must not turn a parked child into a polling child.
14166        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
14167        // a single 60 s jump would only observe one expired timer.
14168        for _ in 0..6000 {
14169            tokio::time::advance(Duration::from_millis(10)).await;
14170            tokio::task::yield_now().await;
14171        }
14172        for actor in [&none, &unregistered, &unadvertised] {
14173            assert!(
14174                actor.turns() <= 5,
14175                "no-health actor woke {} times",
14176                actor.turns()
14177            );
14178            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
14179        }
14180    }
14181
14182    #[tokio::test(start_paused = true)]
14183    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
14184        // With a zero cadence the first probe is due at once, so this measures
14185        // only how fast the registration notification starts health probing,
14186        // not the normal 30-33 s wait before a first probe (checked below).
14187        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
14188        let tick = Instant::now();
14189        actor.hello(1, true);
14190        actor.probe().await;
14191        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
14192    }
14193
14194    #[tokio::test(start_paused = true)]
14195    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
14196        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14197        let hello_tick = Instant::now();
14198        actor.hello(1, true);
14199        let armed = actor
14200            .wait_for_select(|checkpoint| {
14201                checkpoint.registered_connection == Some(ConnectionId::new(1))
14202                    && checkpoint.next_probe_at.is_some()
14203            })
14204            .await;
14205        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14206        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
14207        let first_deadline = hello_tick + delay;
14208        assert_eq!(armed.next_probe_at, Some(first_deadline));
14209        tokio::time::advance(Duration::from_secs(29)).await;
14210        let turns = actor.turns();
14211        actor
14212            .registry
14213            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
14214            .unwrap();
14215        let updated = actor
14216            .wait_for_select(|checkpoint| checkpoint.turn > turns)
14217            .await;
14218        assert_eq!(
14219            updated.next_probe_at,
14220            Some(first_deadline),
14221            "catalog update must not reset cadence"
14222        );
14223        actor.no_probe();
14224        tokio::time::advance(first_deadline - Instant::now()).await;
14225        let frame = actor.probe().await;
14226        actor.answer(1, frame);
14227        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
14228        let next_deadline = Instant::now() + next;
14229        let rearmed = actor
14230            .wait_for_select(|checkpoint| {
14231                checkpoint
14232                    .next_probe_at
14233                    .is_some_and(|deadline| deadline > first_deadline)
14234            })
14235            .await;
14236        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
14237        tokio::time::advance(next - Duration::from_nanos(1)).await;
14238        actor.no_probe();
14239        tokio::time::advance(Duration::from_nanos(1)).await;
14240        actor.probe().await;
14241    }
14242
14243    #[tokio::test(start_paused = true)]
14244    async fn disconnect_disarms_and_reconnect_rearms_health() {
14245        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14246        actor.hello(1, true);
14247        actor
14248            .wait_for_select(|checkpoint| {
14249                checkpoint.registered_connection == Some(ConnectionId::new(1))
14250                    && checkpoint.next_probe_at.is_some()
14251            })
14252            .await;
14253        let turns = actor.turns();
14254        let tick = Instant::now();
14255        actor
14256            .registry
14257            .deregister_connection(ConnectionId::new(1))
14258            .unwrap();
14259        let disarmed = actor
14260            .wait_for_select(|checkpoint| {
14261                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
14262            })
14263            .await;
14264        assert_eq!(disarmed.next_probe_at, None);
14265        assert_eq!(disarmed.wake_after, None);
14266        assert_eq!(
14267            Instant::now(),
14268            tick,
14269            "disconnect must disarm in the notification tick"
14270        );
14271        tokio::time::advance(Duration::from_secs(60)).await;
14272        actor.no_probe();
14273        let reconnect_tick = Instant::now();
14274        actor.hello(2, true);
14275        let rearmed = actor
14276            .wait_for_select(|checkpoint| {
14277                checkpoint.registered_connection == Some(ConnectionId::new(2))
14278                    && checkpoint.next_probe_at.is_some()
14279            })
14280            .await;
14281        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14282        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
14283        tokio::time::advance(delay).await;
14284        actor.probe().await;
14285    }
14286
14287    #[tokio::test(start_paused = true)]
14288    async fn reregistration_without_health_disarms_the_old_schedule() {
14289        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14290        actor.hello(1, true);
14291        actor
14292            .wait_for_select(|checkpoint| {
14293                checkpoint.registered_connection == Some(ConnectionId::new(1))
14294                    && checkpoint.next_probe_at.is_some()
14295            })
14296            .await;
14297        actor
14298            .registry
14299            .deregister_connection(ConnectionId::new(1))
14300            .unwrap();
14301        actor.hello(2, false);
14302        let disarmed = actor
14303            .wait_for_select(|checkpoint| {
14304                checkpoint.registered_connection == Some(ConnectionId::new(2))
14305            })
14306            .await;
14307        assert_eq!(disarmed.next_probe_at, None);
14308        assert_eq!(disarmed.wake_after, None);
14309        tokio::time::advance(Duration::from_secs(60)).await;
14310        actor.no_probe();
14311    }
14312
14313    #[tokio::test(start_paused = true)]
14314    async fn restart_waits_for_its_own_hello_before_rearming_health() {
14315        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14316        actor.hello(1, true);
14317        let armed = actor
14318            .wait_for_select(|checkpoint| {
14319                checkpoint.registered_connection == Some(ConnectionId::new(1))
14320                    && checkpoint.next_probe_at.is_some()
14321            })
14322            .await;
14323        // Simulate the old process's connection closing before the restart
14324        // tears the process down. The replacement process has started but has
14325        // not sent its HELLO, so it is not registered.
14326        actor
14327            .registry
14328            .deregister_connection(ConnectionId::new(1))
14329            .unwrap();
14330        actor.module.restart(Some(0)).await.unwrap();
14331        let parked = actor
14332            .wait_for_select(|checkpoint| {
14333                checkpoint.generation > armed.generation
14334                    && checkpoint.registered_connection.is_none()
14335                    && checkpoint.next_probe_at.is_none()
14336            })
14337            .await;
14338        assert_eq!(parked.wake_after, None);
14339        actor.rx.close();
14340        // Drain the stop notice the restart queued for the old process's connection,
14341        // so it isn't mistaken for traffic to the replacement.
14342        while actor.rx.try_recv().is_ok() {}
14343        let turns = actor.turns();
14344        tokio::time::advance(Duration::from_secs(60)).await;
14345        let after = actor.turns();
14346        eprintln!(
14347            "restart park turns: before={turns} after={after} delta={}",
14348            after - turns
14349        );
14350        assert_eq!(after, turns, "replacement must park before HELLO");
14351        let hello_tick = Instant::now();
14352        actor.hello(2, true);
14353        let rearmed = actor
14354            .wait_for_select(|checkpoint| {
14355                checkpoint.registered_connection == Some(ConnectionId::new(2))
14356                    && checkpoint.next_probe_at.is_some()
14357            })
14358            .await;
14359        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14360        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
14361        tokio::time::advance(delay).await;
14362        actor.probe().await;
14363    }
14364
14365    #[tokio::test(start_paused = true)]
14366    async fn http_probe_waits_a_full_cadence_for_each_spawn() {
14367        use tokio::{
14368            io::{AsyncReadExt, AsyncWriteExt},
14369            net::TcpListener,
14370        };
14371        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14372        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14373        actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
14374        let (started_tx, mut started_rx) = mpsc::channel(8);
14375        let peer = tokio::spawn(async move {
14376            loop {
14377                let (mut socket, _) = listener.accept().await.unwrap();
14378                let mut request = [0; 1024];
14379                assert!(socket.read(&mut request).await.unwrap() > 0);
14380                started_tx.send(Instant::now()).await.unwrap();
14381                socket
14382                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14383                    .await
14384                    .unwrap();
14385            }
14386        });
14387        actor
14388            .module
14389            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14390            .await
14391            .unwrap();
14392        let first_tick = Instant::now();
14393        let first = actor
14394            .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
14395            .await;
14396        assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
14397        assert!(started_rx.try_recv().is_err());
14398
14399        // Restart just before the old deadline: reusing it would give the new
14400        // process only one second to start, rather than a full cadence.
14401        tokio::time::advance(
14402            first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
14403        )
14404        .await;
14405        actor.module.restart(Some(0)).await.unwrap();
14406        let spawn_tick = Instant::now();
14407        let replacement = actor
14408            .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
14409            .await;
14410        let deadline = replacement.next_probe_at.unwrap();
14411        assert!(
14412            deadline >= spawn_tick + actor.health.cadence,
14413            "a replacement must not inherit its predecessor's probe deadline"
14414        );
14415        tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
14416        assert!(
14417            started_rx.try_recv().is_err(),
14418            "no HTTP probe during startup grace"
14419        );
14420        tokio::time::advance(deadline - Instant::now()).await;
14421        let landed = wait_for("first replacement HTTP probe", || {
14422            started_rx.try_recv().ok()
14423        })
14424        .await;
14425        assert!(landed >= spawn_tick + actor.health.cadence);
14426        wait_for("healthy replacement", || {
14427            (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14428                .then_some(())
14429        })
14430        .await;
14431        peer.abort();
14432    }
14433
14434    #[tokio::test(start_paused = true)]
14435    async fn rescan_adds_and_removes_http_health_without_polling() {
14436        use tokio::{
14437            io::{AsyncReadExt, AsyncWriteExt},
14438            net::TcpListener,
14439        };
14440        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14441        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14442        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
14443        // One millisecond separates successive probes without a polling delay.
14444        actor.health.cadence = Duration::from_millis(1);
14445        let (started_tx, mut started_rx) = mpsc::channel(8);
14446        let peer = tokio::spawn(async move {
14447            loop {
14448                let (mut socket, _) = listener.accept().await.unwrap();
14449                let mut request = [0; 1024];
14450                assert!(socket.read(&mut request).await.unwrap() > 0);
14451                started_tx.send(()).await.unwrap();
14452                socket
14453                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14454                    .await
14455                    .unwrap();
14456            }
14457        });
14458        let tick = Instant::now();
14459        let turns = actor.turns();
14460        actor
14461            .module
14462            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14463            .await
14464            .unwrap();
14465        let armed = actor
14466            .wait_for_select(|checkpoint| {
14467                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
14468            })
14469            .await;
14470        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
14471        assert_eq!(Instant::now(), tick);
14472        tokio::time::advance(Duration::from_millis(1)).await;
14473        let mut started = false;
14474        wait_for("successful HTTP probe", || {
14475            started |= started_rx.try_recv().is_ok();
14476            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14477                .then_some(())
14478        })
14479        .await;
14480        assert!(started, "added HTTP check must start at its first deadline");
14481        assert_eq!(
14482            actor.module.status().unwrap().health.status,
14483            SupervisorHealthStatus::Ok
14484        );
14485        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
14486        actor.health.http = None;
14487        let turns = actor.turns();
14488        actor
14489            .module
14490            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14491            .await
14492            .unwrap();
14493        let parked = actor
14494            .wait_for_select(|checkpoint| {
14495                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
14496            })
14497            .await;
14498        assert_eq!(parked.wake_after, None);
14499        let turns = actor.turns();
14500        tokio::time::advance(Duration::from_secs(60)).await;
14501        assert_eq!(
14502            actor.turns(),
14503            turns,
14504            "removed HTTP check must park the actor"
14505        );
14506        assert!(
14507            started_rx.try_recv().is_err(),
14508            "removed HTTP check must stay disarmed"
14509        );
14510        assert_eq!(
14511            actor.module.status().unwrap().health,
14512            ModuleHealthStatus::default()
14513        );
14514        peer.abort();
14515    }
14516}
14517
14518#[cfg(test)]
14519mod drain_busy_reprobe_tests {
14520    //! The drain's busy-gauge wait, driven against an in-process module
14521    //! connection whose health answers the test scripts. Time is paused, so
14522    //! every elapsed duration below is exact virtual time.
14523    use super::*;
14524    use std::collections::BTreeMap;
14525    use std::sync::Mutex as StdMutex;
14526    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14527
14528    const MODULE_ID: &str = "drain-reprobe-module";
14529    const CONNECTION: u64 = 7;
14530    const GAUGE: &str = "runs_in_flight";
14531    /// The production default for both the health cadence and the drain
14532    /// ceiling, which is the pairing that used to cost a full ceiling.
14533    const CADENCE: Duration = Duration::from_secs(30);
14534    const CEILING: Duration = Duration::from_secs(30);
14535
14536    /// How the scripted module answers each `health.check`.
14537    #[derive(Clone, Copy)]
14538    enum Answer {
14539        /// Report the gauge busy until this instant, quiet from then on, after
14540        /// taking `latency` to answer.
14541        BusyUntil { until: Instant, latency: Duration },
14542        /// Answer at once, with metrics that leave the declared gauge out.
14543        OmitsGauge,
14544        /// Never answer.
14545        Silent,
14546    }
14547
14548    struct Drain {
14549        forwarding: Arc<ForwardingTable>,
14550        runtime: SupervisorRuntimeConfig,
14551        endpoint: crate::ModuleEndpointId,
14552        /// When each probe reached the module.
14553        probes: Arc<StdMutex<Vec<Instant>>>,
14554        responder: tokio::task::JoinHandle<()>,
14555    }
14556
14557    impl Drain {
14558        fn start(answer: Answer) -> Self {
14559            let registry = Arc::new(Registry::default());
14560            let forwarding = Arc::new(ForwardingTable::default());
14561            let health = HealthConfig {
14562                cadence: CADENCE,
14563                ..HealthConfig::default()
14564            };
14565            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
14566                .with_forwarding(forwarding.clone())
14567                .with_health_config(health)
14568                .runtime_config();
14569            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
14570            let endpoint = forwarding
14571                .register_module_connection(
14572                    ConnectionId::new(CONNECTION),
14573                    MODULE_ID.to_string(),
14574                    subc_protocol::PROTOCOL_VERSION,
14575                    Concurrency::ModuleManaged,
14576                    FrameSink::new(tx),
14577                )
14578                .unwrap();
14579            let probes = Arc::new(StdMutex::new(Vec::new()));
14580            let responder = tokio::spawn({
14581                let forwarding = forwarding.clone();
14582                let probes = probes.clone();
14583                async move {
14584                    while let Some(frame) = rx.recv().await {
14585                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
14586                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14587                        probes.lock().unwrap().push(Instant::now());
14588                        let metrics = match answer {
14589                            Answer::BusyUntil { until, latency } => {
14590                                sleep(latency).await;
14591                                let busy = u64::from(Instant::now() < until);
14592                                serde_json::json!({ GAUGE: busy })
14593                            }
14594                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
14595                            Answer::Silent => continue,
14596                        };
14597                        // A probe abandoned at its deadline has no RPC left to
14598                        // complete, so a late answer may be refused.
14599                        let _ = forwarding.complete_module_control_rpc(
14600                            ConnectionId::new(CONNECTION),
14601                            frame.header.corr,
14602                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14603                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14604                                status: HealthStatus::Ok,
14605                                detail: None,
14606                                metrics: Some(metrics),
14607                            }),
14608                        );
14609                    }
14610                }
14611            });
14612            Self {
14613                forwarding,
14614                runtime,
14615                endpoint,
14616                probes,
14617                responder,
14618            }
14619        }
14620
14621        /// Run one by-id drain wait with `ceiling`, returning its verdict and
14622        /// how long it took.
14623        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
14624            let (drained, elapsed, _) = self.run_traced(ceiling).await;
14625            (drained, elapsed)
14626        }
14627
14628        /// [`Self::run`], also returning what the drain's probes read.
14629        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
14630            let started = Instant::now();
14631            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
14632            let drained = wait_for_forwarding_quiescence(
14633                &self.forwarding,
14634                MODULE_ID,
14635                &self.runtime,
14636                self.endpoint,
14637                started + ceiling,
14638                &[GAUGE.to_string()],
14639                DrainScope::Active,
14640                &mut trace,
14641            )
14642            .await
14643            .unwrap();
14644            (drained, started.elapsed(), trace)
14645        }
14646
14647        fn probes(&self) -> Vec<Instant> {
14648            self.probes.lock().unwrap().clone()
14649        }
14650    }
14651
14652    impl Drop for Drain {
14653        fn drop(&mut self) {
14654            self.responder.abort();
14655        }
14656    }
14657
14658    #[tokio::test(start_paused = true)]
14659    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
14660        let drain = Drain::start(Answer::BusyUntil {
14661            until: Instant::now() + Duration::from_secs(1),
14662            latency: Duration::ZERO,
14663        });
14664        let (drained, elapsed) = drain.run(CEILING).await;
14665        assert!(
14666            drained,
14667            "gauges that read zero during the drain must let it finish drained, \
14668             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
14669        );
14670        assert!(
14671            elapsed < Duration::from_secs(3),
14672            "drain took {elapsed:?} after the gauges went quiet at 1s"
14673        );
14674        let probes = drain.probes();
14675        assert!(
14676            probes.len() >= 2,
14677            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
14678        );
14679    }
14680
14681    /// The same shape in real time on a multi-threaded runtime: the first
14682    /// probe is answered busy at once, within a millisecond of the drain
14683    /// starting, and later probes read idle. The drain must keep probing
14684    /// after that first instant answer and end soon after the gauge falls.
14685    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14686    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
14687        let drain = Drain::start(Answer::BusyUntil {
14688            until: Instant::now() + Duration::from_millis(600),
14689            latency: Duration::ZERO,
14690        });
14691        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
14692        assert!(drained, "the gauge fell at 600ms: {trace}");
14693        assert!(
14694            elapsed < Duration::from_secs(3),
14695            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
14696        );
14697        let probes = drain.probes();
14698        assert!(
14699            probes.len() >= 2 && trace.probes as usize == probes.len(),
14700            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
14701        );
14702    }
14703
14704    #[tokio::test(start_paused = true)]
14705    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
14706        let started = Instant::now();
14707        let drain = Drain::start(Answer::BusyUntil {
14708            until: started + Duration::from_secs(3600),
14709            latency: Duration::ZERO,
14710        });
14711        let (drained, elapsed) = drain.run(CEILING).await;
14712        assert!(!drained, "a module busy throughout must not drain");
14713        assert!(
14714            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
14715            "a busy module is held until the ceiling and no longer: {elapsed:?}"
14716        );
14717        // Re-probing is paced by the interval, not a busy loop: at most one
14718        // probe per interval across the ceiling, plus the first at time zero.
14719        let probes = drain.probes();
14720        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
14721        assert!(
14722            probes.len() as u128 <= max_probes,
14723            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
14724            probes.len()
14725        );
14726        for pair in probes.windows(2) {
14727            assert!(
14728                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
14729                "consecutive probes {:?} apart",
14730                pair[1] - pair[0]
14731            );
14732        }
14733    }
14734
14735    #[tokio::test(start_paused = true)]
14736    async fn a_slow_probe_is_never_overlapped_by_the_next() {
14737        // Each answer takes longer than the re-probe interval; the next probe
14738        // must wait for the previous answer and then the interval.
14739        let latency = Duration::from_secs(1);
14740        let drain = Drain::start(Answer::BusyUntil {
14741            until: Instant::now() + Duration::from_secs(3600),
14742            latency,
14743        });
14744        let (drained, _) = drain.run(Duration::from_secs(5)).await;
14745        assert!(!drained);
14746        let probes = drain.probes();
14747        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
14748        for pair in probes.windows(2) {
14749            assert!(
14750                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
14751                "a probe started {:?} after the previous one, before it could finish",
14752                pair[1] - pair[0]
14753            );
14754        }
14755    }
14756
14757    #[tokio::test(start_paused = true)]
14758    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
14759        let drain = Drain::start(Answer::Silent);
14760        let ceiling = Duration::from_secs(2);
14761        let (drained, elapsed) = drain.run(ceiling).await;
14762        assert!(!drained, "an unanswered probe counts as busy");
14763        assert!(
14764            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
14765            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
14766        );
14767        assert_eq!(
14768            drain.probes().len(),
14769            1,
14770            "the unanswered probe holds the only slot until the ceiling"
14771        );
14772    }
14773
14774    /// Every event logged while it is installed, as `message` plus fields.
14775    #[derive(Clone, Default)]
14776    struct LogCapture {
14777        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
14778    }
14779
14780    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
14781        fn on_event(
14782            &self,
14783            event: &tracing::Event<'_>,
14784            _context: tracing_subscriber::layer::Context<'_, S>,
14785        ) {
14786            struct Fields(BTreeMap<String, String>);
14787            impl tracing::field::Visit for Fields {
14788                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
14789                    self.0
14790                        .insert(field.name().to_string(), format!("{value:?}"));
14791                }
14792            }
14793            let mut fields = Fields(BTreeMap::new());
14794            event.record(&mut fields);
14795            self.events.lock().unwrap().push(fields.0);
14796        }
14797    }
14798
14799    impl LogCapture {
14800        fn install(&self) -> tracing::subscriber::DefaultGuard {
14801            use tracing_subscriber::layer::SubscriberExt;
14802            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
14803        }
14804
14805        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
14806            self.events
14807                .lock()
14808                .unwrap()
14809                .iter()
14810                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
14811                .cloned()
14812                .collect()
14813        }
14814    }
14815
14816    /// A module that declares a busy gauge but leaves it out of its health
14817    /// answers holds a drain to its ceiling. The timeout line must say so by
14818    /// name, and the drain must warn about the missing gauge once, not on
14819    /// every probe.
14820    #[tokio::test(start_paused = true)]
14821    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
14822        let capture = LogCapture::default();
14823        let _guard = capture.install();
14824        let drain = Drain::start(Answer::OmitsGauge);
14825        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
14826        assert!(!drained, "a gauge never reported cannot read quiet");
14827        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
14828
14829        let missing = capture.with_message("lacks a declared busy gauge");
14830        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
14831        assert_eq!(
14832            missing[0].get("missing_gauges").map(String::as_str),
14833            Some(GAUGE)
14834        );
14835
14836        warn_drain_timed_out(
14837            MODULE_ID,
14838            Duration::from_secs(2),
14839            RouteCloseReason::Restart,
14840            &Default::default(),
14841            &trace,
14842        );
14843        let timeouts = capture.with_message("route drain timed out");
14844        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
14845        let gauges = &timeouts[0]["busy_gauges"];
14846        assert!(
14847            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
14848            "the timeout line must name the gauge and why it read busy: {gauges}"
14849        );
14850        assert!(
14851            gauges.ends_with("; no value ever read"),
14852            "no value was ever read for the gauge: {gauges}"
14853        );
14854        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
14855    }
14856
14857    /// The trace names a busy value and when it was read. For a probe that
14858    /// never got an answer, it names the probe's error kind and says that no
14859    /// value was ever read.
14860    #[tokio::test(start_paused = true)]
14861    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
14862        let busy = Drain::start(Answer::BusyUntil {
14863            until: Instant::now() + Duration::from_secs(3600),
14864            latency: Duration::ZERO,
14865        });
14866        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
14867        let rendered = trace.to_string();
14868        assert!(
14869            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
14870                && rendered.contains("; last value non_zero(1) @"),
14871            "{rendered}"
14872        );
14873
14874        let silent = Drain::start(Answer::Silent);
14875        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
14876        // The probe is capped at the 1 s ceiling, and its reading is stamped
14877        // when it gave up.
14878        assert_eq!(
14879            trace.to_string(),
14880            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
14881        );
14882        assert_eq!(trace.probes, 1);
14883    }
14884
14885    /// A gauge value that is not a count reads busy and is shown as sent.
14886    #[test]
14887    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
14888        let gauges = [GAUGE.to_string()];
14889        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
14890        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
14891        assert_eq!(
14892            busy_gauge_observation(&verdicts),
14893            BusyGaugeObservation::Busy
14894        );
14895        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
14896        assert_eq!(
14897            busy_gauge_observation(&zero),
14898            BusyGaugeObservation::Quiescent
14899        );
14900        let none = gauge_verdicts(None, &gauges);
14901        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
14902    }
14903}
14904
14905#[cfg(test)]
14906mod health_evidence_tests {
14907    use super::{HealthProbeError, HealthProbeEvidence};
14908    use std::collections::HashSet;
14909
14910    /// The evidential asymmetry, asserted rather than described.
14911    ///
14912    /// Exactly ONE observation is proof a module cannot serve, and the one that
14913    /// fires under CPU starvation is not it. Before the split, all fifteen
14914    /// construction sites collapsed into a single String, so a timeout carried the
14915    /// same weight as a dead lane -- which is how a healthy module was restarted
14916    /// three times in one day.
14917    #[test]
14918    fn only_a_dead_lane_is_proof_of_death() {
14919        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
14920        // Three non-proof classes, each for a different reason: silence is
14921        // consistent with health, a bad answer proves the module ALIVE, and a
14922        // daemon-side fault never reached the module at all.
14923        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
14924        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
14925        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
14926    }
14927
14928    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
14929    ///
14930    /// A shared label renders two different observations identically in the line an
14931    /// operator reads after an unexplained restart -- the exact confusion this
14932    /// change removes.
14933    #[test]
14934    fn every_evidence_class_has_a_distinct_label() {
14935        let labels = [
14936            HealthProbeError::lane_dead("").label(),
14937            HealthProbeError::no_answer("").label(),
14938            HealthProbeError::bad_answer("").label(),
14939            HealthProbeError::misconfigured("").label(),
14940        ];
14941        let unique: HashSet<_> = labels.iter().collect();
14942        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
14943    }
14944
14945    /// The class is additional information, not a replacement.
14946    ///
14947    /// An operator needs both "this was silence" and the specific text saying how
14948    /// long we waited; a classification that swallowed the message would trade one
14949    /// missing distinction for another.
14950    #[test]
14951    fn classification_preserves_the_original_message() {
14952        let err = HealthProbeError::no_answer("module did not answer within 5s");
14953        assert_eq!(err.to_string(), "module did not answer within 5s");
14954        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14955    }
14956}
14957
14958#[cfg(test)]
14959mod health_tombstone_tests {
14960    use std::{path::PathBuf, sync::Arc, time::Duration};
14961
14962    use subc_protocol::{
14963        manifest::Concurrency,
14964        session::{HealthStatus, ModuleControlResponse},
14965    };
14966    use tokio::sync::mpsc;
14967
14968    use super::{
14969        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
14970        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
14971    };
14972    use crate::{
14973        control::ControlHandler,
14974        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
14975        registry::{ConnectionId, Registry},
14976        router::FrameSink,
14977    };
14978
14979    struct ProbeHarness {
14980        spec: ModuleSpec,
14981        runtime: SupervisorRuntimeConfig,
14982        forwarding: Arc<ForwardingTable>,
14983        module_connection: ConnectionId,
14984        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14985        handler: ControlHandler,
14986        module: super::SupervisedModule,
14987    }
14988
14989    fn probe_harness() -> ProbeHarness {
14990        let registry = Arc::new(Registry::default());
14991        let forwarding = Arc::new(ForwardingTable::default());
14992        let supervisor_handle = super::SupervisorHandle::new();
14993        let health = HealthConfig {
14994            http: None,
14995            cadence: Duration::from_secs(30),
14996            deadline: Duration::from_secs(5),
14997            failure_threshold: 3,
14998            on_degraded: HealthAction::Report,
14999            on_failing: HealthAction::Report,
15000            critical: false,
15001        };
15002        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
15003            .with_forwarding(Arc::clone(&forwarding))
15004            .with_handle(supervisor_handle.clone())
15005            .with_health_config(health);
15006        let spec = ModuleSpec {
15007            module_id: "late-health-module".to_string(),
15008            program: PathBuf::from("disabled-module"),
15009            args: Vec::new(),
15010            env: Vec::new(),
15011            reserved: false,
15012            reserved_prefixes: Vec::new(),
15013            protocol: ModuleProtocol::Subc,
15014            overlap: Default::default(),
15015        };
15016        let module = supervisor
15017            .supervise_configured(spec.clone(), false)
15018            .unwrap();
15019        let runtime = supervisor.runtime_config();
15020        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
15021            .with_supervisor(supervisor_handle);
15022        let module_connection = ConnectionId::new(700);
15023        let (module_tx, module_rx) = mpsc::channel(8);
15024        forwarding
15025            .register_module_connection(
15026                module_connection,
15027                spec.module_id.clone(),
15028                subc_protocol::PROTOCOL_VERSION,
15029                Concurrency::ModuleManaged,
15030                FrameSink::new(module_tx),
15031            )
15032            .unwrap();
15033
15034        ProbeHarness {
15035            spec,
15036            runtime,
15037            forwarding,
15038            module_connection,
15039            module_rx,
15040            handler,
15041            module,
15042        }
15043    }
15044
15045    async fn finish_after(
15046        harness: &mut ProbeHarness,
15047        stall: Duration,
15048    ) -> ModuleControlRpcCompletion {
15049        assert!(stall > harness.runtime.health.deadline);
15050        let deadline = harness.runtime.health.deadline;
15051        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15052        let answer = async {
15053            let frame = harness.module_rx.recv().await.expect("health.check frame");
15054            tokio::time::advance(deadline).await;
15055            tokio::task::yield_now().await;
15056            tokio::time::advance(stall - deadline).await;
15057            harness
15058                .forwarding
15059                .complete_module_control_rpc(
15060                    harness.module_connection,
15061                    frame.header.corr,
15062                    Some("health.check"),
15063                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15064                        status: HealthStatus::Ok,
15065                        detail: None,
15066                        metrics: None,
15067                    }),
15068                )
15069                .unwrap()
15070        };
15071        let (probe_result, completion) = tokio::join!(probe, answer);
15072        let err = probe_result.expect_err("probe must miss its deadline");
15073        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15074        completion
15075    }
15076
15077    async fn time_out_without_answer(harness: &mut ProbeHarness) {
15078        let deadline = harness.runtime.health.deadline;
15079        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15080        let exhaust_deadline = async {
15081            let _frame = harness.module_rx.recv().await.expect("health.check frame");
15082            tokio::time::advance(deadline).await;
15083            tokio::task::yield_now().await;
15084        };
15085        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
15086        let err = probe_result.expect_err("probe must miss its deadline");
15087        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15088    }
15089
15090    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
15091        let registry = Arc::clone(&harness.module.inner.registry);
15092        let snapshot = Arc::clone(&harness.module.inner.snapshot);
15093        let process_liveness = super::SupervisorProcessLiveness::default();
15094        let mut child = None;
15095        let cycle = super::run_health_probe_cycle(
15096            &harness.spec,
15097            &harness.runtime,
15098            &registry,
15099            &process_liveness,
15100            &snapshot,
15101            &mut child,
15102        );
15103        let peer = async {
15104            let frame = harness.module_rx.recv().await.expect("health.check frame");
15105            if answer {
15106                harness
15107                    .forwarding
15108                    .complete_module_control_rpc(
15109                        harness.module_connection,
15110                        frame.header.corr,
15111                        Some("health.check"),
15112                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15113                            status: HealthStatus::Ok,
15114                            detail: None,
15115                            metrics: Some(serde_json::json!({"ready": true})),
15116                        }),
15117                    )
15118                    .unwrap();
15119            } else {
15120                tokio::time::advance(harness.runtime.health.deadline).await;
15121                tokio::task::yield_now().await;
15122            }
15123        };
15124        tokio::join!(cycle, peer);
15125    }
15126
15127    #[tokio::test(start_paused = true)]
15128    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
15129        let mut harness = probe_harness();
15130        // Drive the probe cycle directly with an in-memory wire peer. Stop the
15131        // disabled module's monitor so only this test owns lifecycle transitions;
15132        // no OS process is launched, and a restart is observed at scheduling.
15133        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
15134        monitor.abort();
15135        let _ = monitor.await;
15136        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
15137            state.enabled = true;
15138            state.state = super::ModuleState::Running;
15139            state.process_alive = true;
15140        })
15141        .unwrap();
15142
15143        run_probe_cycle(&mut harness, true).await;
15144        assert_eq!(
15145            harness.module.status().unwrap().health.status,
15146            super::SupervisorHealthStatus::Ok
15147        );
15148
15149        for failures in 1..harness.runtime.health.failure_threshold {
15150            run_probe_cycle(&mut harness, false).await;
15151            let status = harness.module.status().unwrap();
15152            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15153            assert_eq!(status.health.consecutive_failures, failures);
15154            assert!(status.health.last_probe_ms.is_some());
15155            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
15156            assert!(status.health.metrics.is_none());
15157            assert_eq!(status.state, super::ModuleState::Running);
15158            assert!(status.process_alive);
15159            assert_eq!(status.restart_count, 0);
15160            assert_eq!(status.lifetime_restarts, 0);
15161            assert!(status.health.last_action.is_none());
15162            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15163        }
15164
15165        run_probe_cycle(&mut harness, true).await;
15166        let recovered = harness.module.status().unwrap();
15167        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
15168        assert_eq!(recovered.health.consecutive_failures, 0);
15169        assert!(recovered.health.detail.is_none());
15170        assert_eq!(
15171            recovered.health.metrics,
15172            Some(serde_json::json!({"ready": true}))
15173        );
15174        assert_eq!(recovered.lifetime_restarts, 0);
15175
15176        for failures in 1..=harness.runtime.health.failure_threshold {
15177            run_probe_cycle(&mut harness, false).await;
15178            let status = harness.module.status().unwrap();
15179            assert_eq!(status.health.consecutive_failures, failures);
15180            if failures < harness.runtime.health.failure_threshold {
15181                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15182                assert_eq!(status.state, super::ModuleState::Running);
15183                assert_eq!(status.lifetime_restarts, 0);
15184                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15185            } else {
15186                assert_eq!(
15187                    status.health.status,
15188                    super::SupervisorHealthStatus::Unresponsive
15189                );
15190                assert_eq!(status.state, super::ModuleState::Restarting);
15191                assert_eq!(status.restart_count, 1);
15192                assert_eq!(status.lifetime_restarts, 1);
15193                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
15194                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
15195            }
15196        }
15197    }
15198
15199    #[tokio::test(start_paused = true)]
15200    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
15201        let mut harness = probe_harness();
15202
15203        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
15204        let first_latency = match &first {
15205            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15206            other => panic!("late answer was not retained: {other:?}"),
15207        };
15208        assert!(harness.handler.observe_module_control_completion(first));
15209
15210        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
15211        let second_latency = match &second {
15212            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15213            other => panic!("late answer was not retained: {other:?}"),
15214        };
15215        assert!(harness.handler.observe_module_control_completion(second));
15216
15217        assert_eq!(first_latency, Duration::from_secs(8));
15218        assert_eq!(
15219            second_latency - first_latency,
15220            Duration::from_secs(3),
15221            "latency must grow linearly with the additional stall"
15222        );
15223        let health = harness.module.status().unwrap().health;
15224        assert_eq!(health.late_answer_count, 2);
15225        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
15226    }
15227
15228    /// A module that answers every probe late must never march to the kill
15229    /// threshold: the late answer proves it is alive, so it must clear the miss
15230    /// streak the timeout recorded. Without the reset, a CPU-starved module
15231    /// that serves every probe seconds past the deadline accumulates
15232    /// `consecutive_failures` to the threshold and is killed — the exact
15233    /// sequence from the 2026-08-14 aft disable, where the daemon logged
15234    /// "proves the module is alive" five times while counting five misses.
15235    #[tokio::test(start_paused = true)]
15236    async fn late_answer_clears_the_consecutive_failure_streak() {
15237        let mut harness = probe_harness();
15238
15239        // Timeout recorded first: the probe path saw no answer in time.
15240        time_out_without_answer(&mut harness).await;
15241        harness
15242            .module
15243            .record_health_probe_failure_for_test("[no-answer] test miss")
15244            .unwrap();
15245        assert_eq!(
15246            harness.module.status().unwrap().health.consecutive_failures,
15247            1,
15248            "precondition: the miss must be on the streak before the late answer"
15249        );
15250
15251        // The stalled reply then lands: proof of life.
15252        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
15253        assert!(matches!(
15254            late,
15255            ModuleControlRpcCompletion::LateHealthAnswer { .. }
15256        ));
15257        assert!(harness.handler.observe_module_control_completion(late));
15258
15259        let health = harness.module.status().unwrap().health;
15260        assert_eq!(
15261            health.consecutive_failures, 0,
15262            "a late answer is an answer: the streak must reset"
15263        );
15264        assert_eq!(health.late_answer_count, 1);
15265    }
15266
15267    #[tokio::test(start_paused = true)]
15268    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
15269        let mut harness = probe_harness();
15270
15271        for _ in 0..20 {
15272            time_out_without_answer(&mut harness).await;
15273            assert_eq!(
15274                harness.forwarding.health_probe_tombstone_count().unwrap(),
15275                1
15276            );
15277        }
15278    }
15279}
15280
15281#[cfg(test)]
15282mod child_env_tests {
15283    #[cfg(windows)]
15284    use super::{
15285        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
15286    };
15287    #[cfg(windows)]
15288    use std::sync::{Arc, OnceLock};
15289    #[cfg(windows)]
15290    #[test]
15291    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
15292        let spec = spec(Vec::new());
15293        let mut command = Command::new("unused");
15294        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
15295            .unwrap()
15296            .unwrap();
15297        let name = command
15298            .as_std()
15299            .get_envs()
15300            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
15301            .and_then(|(_, value)| value)
15302            .unwrap();
15303        assert_eq!(name, OsStr::new(handoff.name()));
15304        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
15305        assert!(command.as_std().get_envs().any(|(key, value)| key
15306            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15307            && value == Some(OsStr::new("env"))));
15308        assert!(command
15309            .as_std()
15310            .get_envs()
15311            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
15312    }
15313
15314    #[cfg(windows)]
15315    #[test]
15316    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
15317        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
15318        snapshot.process_alive = true;
15319        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
15320        assert_eq!(
15321            observed_launch_nonce_source(&snapshot).as_deref(),
15322            Some("env")
15323        );
15324        let consumed = Arc::new(OnceLock::new());
15325        snapshot.pipe_consumed = Some(consumed.clone());
15326        assert_eq!(
15327            observed_launch_nonce_source(&snapshot).as_deref(),
15328            Some("env")
15329        );
15330        consumed.set(()).unwrap();
15331        assert_eq!(
15332            observed_launch_nonce_source(&snapshot).as_deref(),
15333            Some("pipe")
15334        );
15335        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
15336        assert_eq!(
15337            observed_launch_nonce_source(&snapshot).as_deref(),
15338            Some("env")
15339        );
15340        clear_current_process_facts(&mut snapshot);
15341        assert_eq!(observed_launch_nonce_source(&snapshot), None);
15342    }
15343    use super::{
15344        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
15345        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
15346        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
15347    };
15348    use std::{ffi::OsStr, path::PathBuf};
15349    use tokio::process::Command;
15350
15351    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
15352        ModuleSpec {
15353            module_id: "env-plan".to_string(),
15354            program: PathBuf::from("/nonexistent"),
15355            args: Vec::new(),
15356            env,
15357            reserved: false,
15358            reserved_prefixes: Vec::new(),
15359            protocol: ModuleProtocol::Subc,
15360            overlap: Default::default(),
15361        }
15362    }
15363
15364    #[test]
15365    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
15366        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
15367        let digest = LaunchSpecDigest::from_spec(&base);
15368        let mut reordered = base.clone();
15369        reordered.env.reverse();
15370        reordered.module_id = "different-id".into();
15371        reordered.reserved = true;
15372        reordered.overlap = Default::default();
15373        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
15374        let mut delimiter_collision = base.clone();
15375        delimiter_collision.env[0] = ("A=B".into(), "C".into());
15376        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
15377        let mut changed = base.clone();
15378        changed.protocol = ModuleProtocol::None;
15379        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15380        changed = base.clone();
15381        changed.args.push("--flag".into());
15382        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15383        changed = base.clone();
15384        changed.program = PathBuf::from("/other");
15385        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15386        changed = base.clone();
15387        changed.env[1].1 = "other".into();
15388        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15389        assert!(!format!("{digest:?}").contains("B=C"));
15390        assert!(!format!("{digest:?}").contains("[u8"));
15391    }
15392
15393    #[cfg(unix)]
15394    #[test]
15395    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
15396        use std::os::unix::ffi::OsStringExt;
15397        let mut left = spec(Vec::new());
15398        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
15399        let mut right = left.clone();
15400        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
15401        assert_eq!(
15402            left.program.to_string_lossy(),
15403            right.program.to_string_lossy()
15404        );
15405        assert_ne!(
15406            LaunchSpecDigest::from_spec(&left),
15407            LaunchSpecDigest::from_spec(&right)
15408        );
15409    }
15410
15411    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
15412    /// one still gets its own.
15413    ///
15414    /// This is the narrow goal `env_clear()` was reached for, and the reason the
15415    /// fix is `env_remove` rather than deleting the line: an operator's ambient
15416    /// filter silently becoming an unconfigured module's log level is a real
15417    /// defect, just a much smaller one than clearing the environment.
15418    ///
15419    /// Asserted on the command plan rather than a spawned child because proving
15420    /// the ABSENCE of an inherited variable needs the parent's environment
15421    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
15422    /// removal as `(key, None)`, which is exactly the distinction wanted: not
15423    /// "absent because nobody set it" but "explicitly unset for the child".
15424    #[test]
15425    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
15426        let mut command = Command::new("/nonexistent");
15427        apply_child_env(&mut command, &spec(Vec::new()));
15428        let removed = command
15429            .as_std()
15430            .get_envs()
15431            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
15432        assert!(
15433            removed,
15434            "ambient CK_LOG must be explicitly removed for an unconfigured module"
15435        );
15436
15437        let mut configured = Command::new("/nonexistent");
15438        apply_child_env(
15439            &mut configured,
15440            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
15441        );
15442        let effective = configured
15443            .as_std()
15444            .get_envs()
15445            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
15446            .last()
15447            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15448        assert_eq!(
15449            effective,
15450            Some(Some("debug".to_string())),
15451            "a module's configured CK_LOG must survive the ambient removal"
15452        );
15453    }
15454
15455    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
15456    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
15457    /// the same reason as the CK_LOG test above.
15458    ///
15459    /// The argument is the load-bearing half: a stock binary exits on an
15460    /// unknown flag before it listens, so with `--subc` appended the mode
15461    /// could not supervise the one process it exists for. Found by the first
15462    /// conformance run (nats-server: `flag provided but not defined: -subc`).
15463    #[test]
15464    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
15465        let connection_file = std::path::Path::new("/run/subc-connection.json");
15466        let handle = SupervisorHandle::new();
15467
15468        let mut none_spec = spec(Vec::new());
15469        none_spec.protocol = ModuleProtocol::None;
15470        let mut none = Command::new("/nonexistent");
15471        let none_handoff =
15472            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
15473                .expect("protocol-none spawn args apply");
15474        assert!(
15475            none_handoff.is_none(),
15476            "protocol:none spawn must not receive a nonce descriptor"
15477        );
15478        #[cfg(windows)]
15479        assert!(!none.as_std().get_envs().any(|(key, value)| key
15480            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15481            && value.is_some()));
15482        assert!(
15483            !none.as_std().get_envs().any(|(key, value)| key
15484                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
15485                && value.is_some()),
15486            "protocol:none spawn must not name a nonce descriptor"
15487        );
15488        let none_args: Vec<String> = none
15489            .as_std()
15490            .get_args()
15491            .map(|a| a.to_string_lossy().into_owned())
15492            .collect();
15493        assert!(
15494            !none_args.iter().any(|a| a == SUBC_ARG),
15495            "protocol:none argv must not carry --subc; got {none_args:?}"
15496        );
15497        let none_has_nonce = none
15498            .as_std()
15499            .get_envs()
15500            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
15501        assert!(
15502            !none_has_nonce,
15503            "protocol:none spawn must not receive a launch nonce"
15504        );
15505        let none_has_module_id = none
15506            .as_std()
15507            .get_envs()
15508            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
15509        assert!(
15510            none_has_module_id,
15511            "SUBC_MODULE_ID is inert and stays on every path"
15512        );
15513        assert!(
15514            handle.spawn_nonce(&none_spec.module_id).is_none(),
15515            "no nonce record for a process that will never present one"
15516        );
15517
15518        // Control: the subc-wire path is unchanged by the branch above.
15519        let wire_spec = spec(Vec::new());
15520        let mut wire = Command::new("/nonexistent");
15521        let wire_handoff =
15522            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
15523                .expect("subc-wire spawn args apply");
15524        let wire_fd_env = wire
15525            .as_std()
15526            .get_envs()
15527            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
15528            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15529        #[cfg(unix)]
15530        assert_eq!(
15531            wire_fd_env,
15532            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
15533            "a subc-wire spawn names the pipe it will receive at descriptor 3"
15534        );
15535        #[cfg(windows)]
15536        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
15537        #[cfg(not(any(unix, windows)))]
15538        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
15539        let wire_args: Vec<String> = wire
15540            .as_std()
15541            .get_args()
15542            .map(|a| a.to_string_lossy().into_owned())
15543            .collect();
15544        assert_eq!(
15545            wire_args,
15546            vec![
15547                SUBC_ARG.to_string(),
15548                connection_file.to_string_lossy().into_owned()
15549            ],
15550            "a subc-wire spawn still carries --subc <path>"
15551        );
15552        assert_eq!(
15553            wire.as_std()
15554                .get_envs()
15555                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
15556            !cfg!(unix),
15557            "only Windows supplies the environment nonce"
15558        );
15559        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
15560    }
15561
15562    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
15563    /// spec tries to set it; only a swap candidate carries it.
15564    ///
15565    /// "Set it only on candidates" is not enough, because spawn applies the
15566    /// spec's env verbatim and the daemon's own environment is inherited: either
15567    /// could hand a plain restart the swap role, and a module reading it would
15568    /// warm on its long swap budget while callers wait. Asserted as an explicit
15569    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
15570    /// test above gives.
15571    #[test]
15572    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
15573        let role = |command: &Command| {
15574            command
15575                .as_std()
15576                .get_envs()
15577                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
15578                .last()
15579                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
15580        };
15581        let forged = spec(vec![(
15582            SUBC_SPAWN_ROLE_ENV.to_string(),
15583            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
15584        )]);
15585
15586        let mut plain = Command::new("/nonexistent");
15587        apply_child_env(&mut plain, &forged);
15588        apply_spawn_role(&mut plain, SpawnRole::Plain);
15589        assert_eq!(
15590            role(&plain),
15591            Some(None),
15592            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
15593        );
15594
15595        let mut candidate = Command::new("/nonexistent");
15596        apply_child_env(&mut candidate, &spec(Vec::new()));
15597        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
15598        assert_eq!(
15599            role(&candidate),
15600            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
15601        );
15602    }
15603
15604    /// Daemon-private capture retention keys never reach the child.
15605    ///
15606    /// cortexkit-log exposes retention as a Rust struct with no environment
15607    /// names, so these entries are supervisor metadata. Passing them through
15608    /// would invent a public child-process contract by accident.
15609    #[test]
15610    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
15611        let mut command = Command::new("/nonexistent");
15612        apply_child_env(
15613            &mut command,
15614            &spec(vec![
15615                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
15616                ("KEPT".to_string(), "yes".to_string()),
15617            ]),
15618        );
15619        let keys: Vec<String> = command
15620            .as_std()
15621            .get_envs()
15622            .filter(|(_, value)| value.is_some())
15623            .map(|(key, _)| key.to_string_lossy().into_owned())
15624            .collect();
15625        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
15626        assert!(
15627            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
15628            "daemon-private capture key leaked to the child: {keys:?}"
15629        );
15630    }
15631}
15632
15633#[cfg(test)]
15634mod jitter_tests {
15635    use super::jittered_health_delay;
15636    use std::{collections::HashSet, time::Duration};
15637
15638    /// Module ids drawn from a real fleet, so the dispersal claim is about names
15639    /// that actually occur rather than invented ones.
15640    ///
15641    /// This is a SAMPLE, not a registry: the property under test is that distinct
15642    /// ids disperse, which holds for any set of distinct strings. Several entries
15643    /// are already historical (modules get renamed), and that costs nothing here --
15644    /// but it means a reader must not mistake this for the live module set, and a
15645    /// rename sweep will match it without there being anything to change.
15646    const FLEET: [&str; 14] = [
15647        "aft",
15648        "alfonso-core",
15649        "magic-context",
15650        "broca",
15651        "thalamus",
15652        "quota",
15653        "engram",
15654        "plexus",
15655        "cerebellum",
15656        "astrocyte",
15657        "synapse",
15658        "subc-mcp",
15659        "cortexkit-credentials",
15660        "subc-federation",
15661    ];
15662
15663    /// Probes must not converge after a fleet-wide restart.
15664    ///
15665    /// This is the property the jitter exists for: every module reconnects at
15666    /// once, and without dispersal all fourteen would then probe on the same
15667    /// tick forever. Nothing failed visibly when this went untested -- a
15668    /// convergent fleet still probes correctly, just in a burst, so the symptom
15669    /// is a periodic load spike that looks like whatever else is running.
15670    #[test]
15671    fn probe_delays_disperse_across_the_fleet() {
15672        let cadence = Duration::from_secs(30);
15673        let delays: HashSet<Duration> = FLEET
15674            .iter()
15675            .map(|id| jittered_health_delay(id, 0, cadence))
15676            .collect();
15677        assert_eq!(
15678            delays.len(),
15679            FLEET.len(),
15680            "every supervised module must land on its own probe offset"
15681        );
15682    }
15683
15684    /// The offset may only ever DELAY a probe, never bring it forward.
15685    ///
15686    /// A delay below the cadence would probe a module more often than
15687    /// configured, which is the opposite of what an operator asked for and
15688    /// would tighten the failure budget without anyone changing it.
15689    #[test]
15690    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
15691        let cadence = Duration::from_secs(30);
15692        let span = cadence / 10;
15693        for id in FLEET {
15694            for probe_index in 0..8 {
15695                let delay = jittered_health_delay(id, probe_index, cadence);
15696                assert!(
15697                    delay >= cadence,
15698                    "{id}#{probe_index}: jitter must not shorten the cadence"
15699                );
15700                assert!(
15701                    delay < cadence + span,
15702                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
15703                );
15704            }
15705        }
15706    }
15707
15708    /// A module keeps its offset across daemon restarts.
15709    ///
15710    /// The delay is derived rather than randomised precisely so a restart does
15711    /// not re-roll every module into a fresh chance of collision. A random
15712    /// source would satisfy the dispersal test above and quietly lose this.
15713    #[test]
15714    fn a_module_offset_is_stable_across_restarts() {
15715        let cadence = Duration::from_secs(30);
15716        for id in FLEET {
15717            assert_eq!(
15718                jittered_health_delay(id, 0, cadence),
15719                jittered_health_delay(id, 0, cadence),
15720                "{id}: the same module and probe index must produce the same offset"
15721            );
15722        }
15723    }
15724
15725    /// A zero cadence disables probing rather than producing a busy loop.
15726    #[test]
15727    fn zero_cadence_yields_zero_delay() {
15728        assert_eq!(
15729            jittered_health_delay("aft", 0, Duration::ZERO),
15730            Duration::ZERO
15731        );
15732    }
15733}
15734
15735#[cfg(all(test, target_os = "linux"))]
15736mod cgroup_placement_tests {
15737    use super::{
15738        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
15739        SuperviseError, SupervisedChild,
15740    };
15741    use crate::stderr_tail::{StderrRing, StderrTailConfig};
15742    use cortexkit_test_support::ScratchDir;
15743    use std::{
15744        fs, io,
15745        path::{Path, PathBuf},
15746        sync::{Arc, Mutex},
15747    };
15748    use tokio::process::Command;
15749
15750    #[tokio::test]
15751    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
15752        use super::*;
15753        let dir = ScratchDir::new("unique-spawn-cgroups");
15754        let root = PathBuf::from(format!(
15755            "/sys/fs/cgroup/subc-unique-test-{}-{}",
15756            std::process::id(),
15757            unix_ms_now()
15758        ));
15759        if let Err(error) = fs::create_dir(&root) {
15760            assert!(
15761                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15762                "required cgroup test cannot execute: {error}"
15763            );
15764            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
15765            return;
15766        }
15767        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
15768        let group_count = || {
15769            fs::read_dir(root.join("subc-modules"))
15770                .unwrap()
15771                .map(|entry| entry.unwrap().file_type().unwrap())
15772                .filter(|kind| kind.is_dir())
15773                .count()
15774        };
15775        let supervisor =
15776            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15777                .with_cgroup_placement(Some(placement.clone()));
15778        let runtime = supervisor.runtime_config();
15779        let mut spec = ModuleSpec {
15780            module_id: "unique-spawn".into(),
15781            program: PathBuf::from("/bin/sleep"),
15782            args: vec!["60".into()],
15783            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15784                .into_iter()
15785                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
15786                .collect(),
15787            reserved: false,
15788            reserved_prefixes: vec![],
15789            protocol: ModuleProtocol::None,
15790            overlap: Default::default(),
15791        };
15792        let spawn = |spec: &ModuleSpec| {
15793            spawn_child(
15794                spec,
15795                None,
15796                None,
15797                &runtime.stderr_ring,
15798                None,
15799                &runtime.child_roster,
15800                Some(&placement),
15801            )
15802            .unwrap()
15803        };
15804        let mut live = spawn(&spec);
15805        for _ in 0..3 {
15806            // A new process can enter the old slot while retirement is pending.
15807            let next = spawn(&spec);
15808            assert_ne!(live.module_id, next.module_id);
15809            live.start_kill().unwrap();
15810            live.wait().await.unwrap();
15811            live = next;
15812            assert!(
15813                live.child.try_wait().unwrap().is_none(),
15814                "retiring the old slot must not kill the replacement"
15815            );
15816            assert_eq!(
15817                group_count(),
15818                1,
15819                "only the live spawn's cgroup should remain"
15820            );
15821        }
15822        supervisor.begin_daemon_shutdown();
15823        let reap = tokio::spawn(async move {
15824            live.wait().await.unwrap();
15825        });
15826        supervisor
15827            .end_children_for_daemon_shutdown(false, std::future::pending())
15828            .await;
15829        reap.await.unwrap();
15830        assert_eq!(group_count(), 0);
15831        // A normal exit uses the same tree-cleanup path as a killed spawn.
15832        spec.program = PathBuf::from("/bin/true");
15833        spec.args.clear();
15834        let fresh_roster = ChildRoster::default();
15835        let mut short = spawn_child(
15836            &spec,
15837            None,
15838            None,
15839            &runtime.stderr_ring,
15840            None,
15841            &fresh_roster,
15842            Some(&placement),
15843        )
15844        .unwrap();
15845        short.wait().await.unwrap();
15846        assert_eq!(group_count(), 0);
15847        spec.module_id = "_".repeat(255);
15848        let mut long_id = spawn_child(
15849            &spec,
15850            None,
15851            None,
15852            &runtime.stderr_ring,
15853            None,
15854            &fresh_roster,
15855            Some(&placement),
15856        )
15857        .unwrap();
15858        long_id.wait().await.unwrap();
15859        assert_eq!(
15860            group_count(),
15861            0,
15862            "valid long module IDs must not exceed cgroup NAME_MAX"
15863        );
15864        fs::remove_dir(root.join("subc-modules")).unwrap();
15865        fs::remove_dir(root).unwrap();
15866        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
15867    }
15868
15869    #[test]
15870    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
15871        let path = Path::new("/definitely-missing-subc-cgroup");
15872        let mut command = Command::new("true");
15873        let error = apply_cgroup_placement(
15874            &mut command,
15875            &ModuleSpec {
15876                module_id: "broken-cgroup".to_string(),
15877                program: PathBuf::from("true"),
15878                args: Vec::new(),
15879                env: Vec::new(),
15880                reserved: false,
15881                reserved_prefixes: Vec::new(),
15882                protocol: ModuleProtocol::Subc,
15883                overlap: Default::default(),
15884            },
15885            path,
15886        )
15887        .expect_err("a parent cgroup open failure must reject the supervised spawn");
15888        let reason = error.to_string();
15889
15890        assert!(
15891            matches!(error, SuperviseError::Cgroup { .. }),
15892            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
15893        );
15894        assert!(
15895            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
15896            "parent cgroup open failure must name cgroup.procs: {reason}"
15897        );
15898    }
15899
15900    #[tokio::test]
15901    async fn reaping_a_child_removes_its_empty_module_cgroup() {
15902        let root = ScratchDir::new("supervisor-reap-cgroup");
15903        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15904        let placement = subc_cgroup::prepare_at(&root)
15905            .expect("prepare scratch cgroup root")
15906            .expect("scratch root has a cgroup.procs marker");
15907        let module_id = "reaped-module";
15908        let module = placement
15909            .module_path(module_id)
15910            .expect("create scratch module cgroup");
15911        let child = Command::new("true")
15912            .env("XDG_DATA_HOME", root.path())
15913            .env("XDG_RUNTIME_DIR", root.path())
15914            .env("XDG_CONFIG_HOME", root.path())
15915            .spawn()
15916            .expect("spawn short-lived child");
15917        let pid = child.id().expect("spawned child has pid");
15918        let mut child = SupervisedChild {
15919            child,
15920            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
15921            protocol: ModuleProtocol::Subc,
15922            module_id: module_id.to_string(),
15923            cgroup_placement: Some(placement),
15924            stdout_pump: None,
15925            stderr_pump: None,
15926            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
15927            spawned_at: tokio::time::Instant::now(),
15928            spawned_at_ms: 0,
15929            spawned_from: PathBuf::from("true"),
15930            launch_spec_digest: LaunchSpecDigest([0; 32]),
15931            spawned_file_identity: None,
15932            process_start_time: None,
15933            process_identity: None,
15934            pid,
15935            roster_guard: None,
15936            #[cfg(target_os = "macos")]
15937            privacy_exec: None,
15938            spawn_failure: None,
15939        };
15940
15941        child.wait().await.expect("reap short-lived child");
15942
15943        assert!(
15944            !module.exists(),
15945            "reaping the supervised child must remove its empty cgroup"
15946        );
15947    }
15948
15949    #[test]
15950    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
15951        let root = ScratchDir::new("supervisor-non-empty-cgroup");
15952        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15953        let placement = subc_cgroup::prepare_at(&root)
15954            .expect("prepare scratch cgroup root")
15955            .expect("scratch root has a cgroup.procs marker");
15956        let module = placement
15957            .module_path("surviving-module")
15958            .expect("create scratch module cgroup");
15959        fs::write(module.join("surviving-process"), b"still present")
15960            .expect("make scratch cgroup non-empty");
15961        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
15962
15963        remove_module_cgroup(&placement, "surviving-module");
15964
15965        let logs = crate::router::test_log::captured_logs(&logs);
15966        assert!(
15967            module.exists(),
15968            "failed removal must leave the cgroup intact"
15969        );
15970        assert!(
15971            logs.contains("could not remove module cgroup after process exit; continuing teardown")
15972                && logs.contains("surviving-module"),
15973            "best-effort removal must report the failure without returning it: {logs}"
15974        );
15975    }
15976
15977    #[test]
15978    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
15979        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
15980        let reason = SuperviseError::Spawn {
15981            program: PathBuf::from("/bin/true"),
15982            source: io::Error::from_raw_os_error(13),
15983            cgroup_path: Some(cgroup_path.clone()),
15984        }
15985        .to_string();
15986
15987        assert!(
15988            reason.contains(&cgroup_path.display().to_string()),
15989            "a pre_exec spawn failure must name the cgroup path: {reason}"
15990        );
15991    }
15992}
15993
15994#[cfg(test)]
15995mod spawn_subscriber_lag_tests {
15996    use super::*;
15997
15998    /// A subscriber whose connection stops draining is dropped once its frame
15999    /// channel fills. The client must learn that from a terminal Error frame
16000    /// after the frames already queued for it, not from a stream that simply
16001    /// goes quiet.
16002    #[tokio::test]
16003    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
16004        let feed = SpawnEventFeed::default();
16005        feed.configure_incarnation("lag-incarnation".to_string());
16006        // A one-slot connection queue that nobody reads until the emits are
16007        // done: the forwarder parks on it and the subscriber channel fills.
16008        let (tx, mut rx) = mpsc::channel(1);
16009        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
16010            .expect("subscribe");
16011        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
16012        for index in 0..emitted {
16013            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
16014            // Let the forwarder take what it can so the fill point is the
16015            // subscriber channel, not a scheduling accident.
16016            tokio::task::yield_now().await;
16017        }
16018        assert_eq!(
16019            feed.subscriber_count(),
16020            0,
16021            "the lagged subscriber must be removed"
16022        );
16023
16024        let mut data = Vec::new();
16025        let mut last = None;
16026        loop {
16027            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16028                .await
16029                .expect("the forwarder must finish once the subscriber is dropped");
16030            let Some(outbound) = next else { break };
16031            let frame = outbound.frame;
16032            if frame.header.ty == FrameType::StreamData {
16033                assert!(last.is_none(), "no data may follow the terminal frame");
16034                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
16035                data.push(event.cursor.seq);
16036            } else {
16037                assert!(last.is_none(), "exactly one terminal frame");
16038                last = Some(frame);
16039            }
16040        }
16041        assert!(!data.is_empty(), "queued frames drain before the terminal");
16042        for pair in data.windows(2) {
16043            assert_eq!(
16044                pair[1],
16045                pair[0] + 1,
16046                "queued frames arrive dense and in order"
16047            );
16048        }
16049        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
16050        assert_eq!(terminal.header.ty, FrameType::Error);
16051        assert_eq!(terminal.header.corr, 7);
16052        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
16053        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
16054        let detail = body.detail.expect("lagged error carries detail");
16055        assert_eq!(
16056            detail["first_undelivered_cursor"]["seq"],
16057            data.last().unwrap() + 1,
16058            "the named cursor is the first event the subscriber did not receive"
16059        );
16060        assert_eq!(
16061            detail["first_undelivered_cursor"]["daemon_incarnation"],
16062            "lag-incarnation"
16063        );
16064    }
16065}
16066
16067#[cfg(test)]
16068mod terminal_history_read_concurrency_tests {
16069    use super::*;
16070    use crate::terminal_journal::read_pause;
16071    use cortexkit_test_support::ScratchDir;
16072    use std::sync::mpsc as std_mpsc;
16073
16074    fn journaled_ring(
16075        journal: &Arc<crate::terminal_journal::TerminalJournal>,
16076    ) -> Arc<Mutex<TerminalRing>> {
16077        Arc::new(Mutex::new(
16078            TerminalRing::new(TerminalRingConfig::default(), 1)
16079                .with_journal(Some(Arc::clone(journal))),
16080        ))
16081    }
16082
16083    fn crash(at_ms: u64) -> ExitReport {
16084        ExitReport {
16085            kind: ExitKind::Crash,
16086            code: Some(1),
16087            signal: None,
16088            at_ms,
16089        }
16090    }
16091
16092    /// Record an exit on another thread and report whether it finished within
16093    /// `bound`. The recorder thread is left running if it did not.
16094    fn record_within(
16095        module_id: &'static str,
16096        ring: &Arc<Mutex<TerminalRing>>,
16097        at_ms: u64,
16098        bound: Duration,
16099    ) -> bool {
16100        let ring = Arc::clone(ring);
16101        let (done, done_rx) = std_mpsc::channel();
16102        std::thread::spawn(move || {
16103            record_terminal(
16104                module_id,
16105                &ring,
16106                &SpawnEventFeed::default(),
16107                &crash(at_ms),
16108                TerminalDisposition::Restarting,
16109            );
16110            let _ = done.send(());
16111        });
16112        done_rx.recv_timeout(bound).is_ok()
16113    }
16114
16115    /// A history read in progress must not hold the journal writer (which every
16116    /// module's exit recording needs) or the module's own ring. Exits recorded
16117    /// while the read is paused complete promptly; the paused read answers as of
16118    /// the moment it started, and the next read has each exit exactly once.
16119    #[test]
16120    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
16121        let dir = ScratchDir::new("terminal-history-concurrent-read");
16122        let path = dir.join("terminals.jsonl");
16123        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
16124            path.clone(),
16125            "daemon".into(),
16126        ));
16127        let reader_ring = journaled_ring(&journal);
16128        let other_ring = journaled_ring(&journal);
16129        assert!(record_within(
16130            "reader-module",
16131            &reader_ring,
16132            10,
16133            Duration::from_secs(5)
16134        ));
16135
16136        let (started, release) = read_pause::install(&path);
16137        let reading = {
16138            let ring = Arc::clone(&reader_ring);
16139            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
16140        };
16141        started
16142            .recv_timeout(Duration::from_secs(5))
16143            .expect("the history read reached its pause");
16144
16145        let bound = Duration::from_secs(1);
16146        assert!(
16147            record_within("other-module", &other_ring, 20, bound),
16148            "another module's exit waited on a history read (journal writer held)"
16149        );
16150        assert!(
16151            record_within("reader-module", &reader_ring, 30, bound),
16152            "the read module's own exit waited on its history read (ring held)"
16153        );
16154
16155        drop(release);
16156        let paused = reading.join().unwrap();
16157        assert_eq!(
16158            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16159            vec![10],
16160            "an exit recorded after the read began lands in neither half of it"
16161        );
16162        assert_eq!(paused.journal_skipped_lines, 0);
16163        assert_eq!(paused.journal_read_errors, 0);
16164
16165        let after = durable_terminal_history_of(&reader_ring, "reader-module");
16166        assert_eq!(
16167            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16168            vec![10, 30],
16169            "the next read merges ring and journal with no duplicate"
16170        );
16171        assert_eq!(after.journal_skipped_lines, 0);
16172    }
16173}
16174
16175/// What a restart does with the exited process's output readers. These drive
16176/// the same settle functions the supervisor calls, with a reader the test
16177/// holds, so a reader that has not been scheduled by the bound is a controlled
16178/// input rather than something only a loaded machine produces.
16179#[cfg(test)]
16180mod stderr_settle_tests {
16181    use std::{
16182        future::Future,
16183        io,
16184        pin::Pin,
16185        sync::{Arc, Mutex},
16186        task::{Context, Poll},
16187        time::Duration,
16188    };
16189
16190    use tokio::{
16191        io::{AsyncRead, ReadBuf},
16192        sync::oneshot,
16193        time::Instant,
16194    };
16195
16196    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
16197    use crate::stderr_tail::{
16198        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
16199        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
16200    };
16201
16202    const BOUND: Duration = Duration::from_millis(250);
16203
16204    /// Yields `before`, then stays pending until the gate is released, then
16205    /// yields `after` and reaches EOF. The bytes after the gate were written
16206    /// by a process that has already exited; only the reader is behind.
16207    struct HeldReader {
16208        before: Option<Vec<u8>>,
16209        gate: Option<oneshot::Receiver<()>>,
16210        after: io::Cursor<Vec<u8>>,
16211    }
16212
16213    impl AsyncRead for HeldReader {
16214        fn poll_read(
16215            mut self: Pin<&mut Self>,
16216            cx: &mut Context<'_>,
16217            buf: &mut ReadBuf<'_>,
16218        ) -> Poll<io::Result<()>> {
16219            if let Some(bytes) = self.before.take() {
16220                buf.put_slice(&bytes);
16221                return Poll::Ready(Ok(()));
16222            }
16223            if let Some(gate) = self.gate.as_mut() {
16224                match Pin::new(gate).poll(cx) {
16225                    Poll::Pending => return Poll::Pending,
16226                    Poll::Ready(_) => self.gate = None,
16227                }
16228            }
16229            Pin::new(&mut self.after).poll_read(cx, buf)
16230        }
16231    }
16232
16233    struct DiscardSink;
16234
16235    impl OutputSink for DiscardSink {
16236        fn write_line(&mut self, _line: &[u8]) {}
16237    }
16238
16239    fn line(text: &str) -> TailEntry {
16240        TailEntry::Line {
16241            text: text.to_string(),
16242            truncated: false,
16243            at_ms: None,
16244        }
16245    }
16246
16247    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
16248        ring.lock().unwrap()
16249    }
16250
16251    /// Start a reader for a new process generation that delivers `before`
16252    /// immediately and `after` only once the returned sender fires (or is
16253    /// dropped).
16254    fn held_pump(
16255        ring: &Arc<Mutex<StderrRing>>,
16256        before: &str,
16257        after: &str,
16258    ) -> (StderrPump, oneshot::Sender<()>) {
16259        let generation = lock(ring).begin_process();
16260        let (release, gate) = oneshot::channel();
16261        let reader = HeldReader {
16262            before: Some(before.as_bytes().to_vec()),
16263            gate: Some(gate),
16264            after: io::Cursor::new(after.as_bytes().to_vec()),
16265        };
16266        let task = tokio::spawn(pump_stderr_to(
16267            reader,
16268            Arc::clone(ring),
16269            generation,
16270            DiscardSink,
16271        ));
16272        (StderrPump { task, generation }, release)
16273    }
16274
16275    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
16276        for _ in 0..1000 {
16277            if done(&lock(ring)) {
16278                return;
16279            }
16280            tokio::time::sleep(Duration::from_millis(1)).await;
16281        }
16282        panic!(
16283            "ring never reached the expected state: {:?}",
16284            lock(ring).snapshot(None, None)
16285        );
16286    }
16287
16288    #[tokio::test(start_paused = true)]
16289    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
16290        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16291        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
16292
16293        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
16294        let before_release = lock(&ring).snapshot(None, None);
16295        assert!(
16296            matches!(before_release.capture, CaptureState::Incomplete { .. }),
16297            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
16298        );
16299
16300        // The restart: the next process starts and writes before the old
16301        // reader catches up.
16302        let next = lock(&ring).begin_process();
16303        lock(&ring).push_line_from(next, "next process booting");
16304        release.send(()).unwrap();
16305        wait_until(&ring, |ring| {
16306            ring.snapshot(None, None).capture == CaptureState::Captured
16307        })
16308        .await;
16309
16310        assert_eq!(
16311            untimed(lock(&ring).snapshot(None, None).entries),
16312            vec![
16313                line("booting"),
16314                line("config error: missing storage"),
16315                TailEntry::ProcessStart,
16316                line("next process booting"),
16317            ],
16318            "the crash's last line must survive a slow reader and stay in the crashed process's section"
16319        );
16320    }
16321
16322    #[tokio::test(start_paused = true)]
16323    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
16324    ) {
16325        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16326        // `_held` is never fired: a descendant keeps the pipe open for the
16327        // whole test.
16328        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
16329
16330        let started = Instant::now();
16331        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
16332        assert_eq!(
16333            started.elapsed(),
16334            BOUND,
16335            "the restart must wait exactly the bound for a pipe that stays open, no longer"
16336        );
16337
16338        let next = lock(&ring).begin_process();
16339        lock(&ring).push_line_from(next, "next process booting");
16340        tokio::time::sleep(Duration::from_secs(60)).await;
16341
16342        let snapshot = lock(&ring).snapshot(None, None);
16343        match &snapshot.capture {
16344            CaptureState::Incomplete { reason } => assert!(
16345                reason.contains("had not reached EOF") && reason.contains("250ms"),
16346                "the reason must say what is missing and after how long: {reason}"
16347            ),
16348            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
16349        }
16350        assert_eq!(
16351            untimed(snapshot.entries),
16352            vec![
16353                line("parent exiting"),
16354                TailEntry::ProcessStart,
16355                line("next process booting"),
16356            ]
16357        );
16358    }
16359
16360    #[tokio::test(start_paused = true)]
16361    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
16362        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16363        let (pump, release) = held_pump(&ring, "one\n", "two\n");
16364        release.send(()).unwrap();
16365
16366        settle_stderr_pump("clean", &ring, pump, BOUND).await;
16367
16368        let snapshot = lock(&ring).snapshot(None, None);
16369        assert_eq!(snapshot.capture, CaptureState::Captured);
16370        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
16371    }
16372
16373    #[tokio::test(start_paused = true)]
16374    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
16375        const BURST: usize = 150;
16376        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
16377        let path = capture.join("two-pipe.stderr.log");
16378        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
16379        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16380        let generation = lock(&ring).begin_process();
16381        let stdout = (0..BURST)
16382            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
16383            .collect::<String>();
16384        let stderr = (0..BURST)
16385            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
16386            .collect::<String>();
16387        // Model a child that has written both bursts and exited. Hold the stdout
16388        // reader after one line and a partial prefix, as if pipe readiness or
16389        // capture-file writes had delayed it. The final line also needs EOF to
16390        // flush: neither complete nor partial buffered output may be discarded.
16391        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
16392        let (release, gate) = oneshot::channel();
16393        let reader = HeldReader {
16394            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
16395            gate: Some(gate),
16396            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
16397        };
16398        let (finished, completion) = oneshot::channel();
16399        let stdout_sink = sink.clone();
16400        let pump = tokio::spawn(async move {
16401            pump_stdout_to(reader, stdout_sink).await;
16402            finished.send(()).unwrap();
16403        });
16404        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
16405
16406        let started = Instant::now();
16407        settle_stdout_pump("two-pipe", pump, BOUND).await;
16408        assert_eq!(
16409            started.elapsed(),
16410            BOUND,
16411            "a late reader must not hold up restart"
16412        );
16413        assert_eq!(
16414            std::fs::read_to_string(&path).unwrap().lines().count(),
16415            BURST + 1
16416        );
16417
16418        release
16419            .send(())
16420            .expect("stdout reader must survive the exit drain bound");
16421        tokio::time::timeout(Duration::from_secs(1), completion)
16422            .await
16423            .expect("released stdout reader must reach EOF")
16424            .expect("stdout capture must finish at EOF, not be cancelled");
16425        let contents = std::fs::read_to_string(&path).unwrap();
16426        let mut lines = contents
16427            .lines()
16428            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
16429            .collect::<Vec<_>>();
16430        let mut expected = (0..BURST)
16431            .flat_map(|i| {
16432                [
16433                    format!("out-{i:04}-{}", "o".repeat(64)),
16434                    format!("err-{i:04}-{}", "e".repeat(64)),
16435                ]
16436            })
16437            .collect::<Vec<_>>();
16438        lines.sort_unstable();
16439        expected.sort_unstable();
16440        assert_eq!(
16441            lines, expected,
16442            "both pipes must retain every complete line exactly once"
16443        );
16444    }
16445}
16446
16447/// Containment of a module's process tree (issue #109).
16448///
16449/// The behaviour these defend against is a module helper surviving its module:
16450/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
16451/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
16452/// compounds it.
16453///
16454/// They run against the SUPERVISOR rather than the job-object crate because the
16455/// claim is about teardown: a crate-level test proves a job can reap a tree, not
16456/// that the daemon's drain path reaches it.
16457///
16458/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
16459/// lane there is a separate containment path with its own tests.
16460#[cfg(all(test, windows))]
16461mod job_containment_tests {
16462    use super::*;
16463    use cortexkit_test_support::ScratchDir;
16464    use std::{
16465        path::{Path, PathBuf},
16466        sync::{Arc, Mutex},
16467        time::{Duration, Instant},
16468    };
16469
16470    /// The stub, expected beside this test executable.
16471    ///
16472    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
16473    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
16474    /// failure then reads as a broken test rather than an unbuilt dependency.
16475    fn stub_path() -> PathBuf {
16476        let mut path = std::env::current_exe().expect("current_exe available in tests");
16477        path.pop();
16478        path.pop();
16479        path.push("fake-aft-stub.exe");
16480        assert!(
16481            path.exists(),
16482            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
16483             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
16484            path.display()
16485        );
16486        path
16487    }
16488
16489    /// Poll for the grandchild pid the stub records, and parse it.
16490    fn read_grandchild_pid(path: &Path) -> u32 {
16491        let deadline = Instant::now() + Duration::from_secs(10);
16492        loop {
16493            if let Ok(contents) = std::fs::read_to_string(path) {
16494                if let Ok(pid) = contents.trim().parse() {
16495                    return pid;
16496                }
16497            }
16498            assert!(
16499                Instant::now() < deadline,
16500                "the stub never recorded a grandchild pid at {}",
16501                path.display()
16502            );
16503            std::thread::sleep(Duration::from_millis(10));
16504        }
16505    }
16506
16507    /// Everything one fixture run needs, so the two tests below differ in exactly
16508    /// one place: whether the child is contained.
16509    struct Fixture {
16510        _dir: ScratchDir,
16511        module_id: String,
16512        grandchild: u32,
16513        child: Option<SupervisedChild>,
16514        registry: Arc<Registry>,
16515        snapshot: Arc<Mutex<SupervisorSnapshot>>,
16516        terminal_ring: Arc<Mutex<TerminalRing>>,
16517        spawn_events: SpawnEventFeed,
16518    }
16519
16520    fn fixture(label: &str, module_id: &str) -> Fixture {
16521        let dir = ScratchDir::new(label);
16522        let pid_file = dir.join("grandchild.pid");
16523        let supervisor = Supervisor::new_for_test(
16524            Arc::new(Registry::default()),
16525            RestartPolicy::new(3, Duration::ZERO),
16526        );
16527        let runtime = supervisor.runtime_config();
16528        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16529        let spec = ModuleSpec {
16530            module_id: module_id.to_string(),
16531            program: stub_path(),
16532            // Zero args deliberately: a `--subc` argument would make the stub dial
16533            // a daemon that is not there, and the failure would land in the same
16534            // stderr ring this fixture exists to keep quiet.
16535            args: Vec::new(),
16536            env: vec![
16537                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16538                (
16539                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
16540                    pid_file.display().to_string(),
16541                ),
16542            ],
16543            reserved: false,
16544            reserved_prefixes: Vec::new(),
16545            protocol: ModuleProtocol::Subc,
16546            overlap: Default::default(),
16547        };
16548        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
16549            .expect("spawn the supervised fixture");
16550        let grandchild = read_grandchild_pid(&pid_file);
16551        Fixture {
16552            _dir: dir,
16553            module_id: module_id.to_string(),
16554            grandchild,
16555            child: Some(child),
16556            registry: Arc::new(Registry::default()),
16557            snapshot,
16558            terminal_ring: Arc::clone(&runtime.terminal_ring),
16559            spawn_events: SpawnEventFeed::default(),
16560        }
16561    }
16562
16563    impl Fixture {
16564        /// Drain through the supervisor's own teardown path.
16565        async fn drain(&mut self) {
16566            let child = self
16567                .child
16568                .take()
16569                .expect("the fixture child is still present");
16570            drain_child_to_state(
16571                &self.module_id,
16572                ModuleProtocol::Subc,
16573                // No forwarding table in this fixture, so nothing reaches the
16574                // child over a connection.
16575                StopNotice::NotSent,
16576                &self.registry,
16577                None,
16578                &self.snapshot,
16579                &self.terminal_ring,
16580                &self.spawn_events,
16581                child,
16582                Duration::from_millis(500),
16583                ModuleState::Stopped,
16584                Some(false),
16585                &FailureContext::operator(),
16586            )
16587            .await
16588            .expect("drain the supervised fixture");
16589        }
16590    }
16591
16592    /// Teardown reaps the grandchild, not merely the direct child.
16593    ///
16594    /// This is the assertion the change exists for. Before containment the
16595    /// grandchild survived: it is a separate process, and `start_kill` is
16596    /// `TerminateProcess` scoped to one pid.
16597    #[tokio::test]
16598    async fn teardown_reaps_the_grandchild() {
16599        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
16600        let grandchild = fixture.grandchild;
16601
16602        assert!(
16603            subc_jobobject::process_exists(grandchild),
16604            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
16605        );
16606
16607        fixture.drain().await;
16608
16609        assert!(
16610            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16611            "grandchild {grandchild} outlived module teardown: the tree was not contained"
16612        );
16613    }
16614
16615    /// The mutation control: with containment withheld, the grandchild survives
16616    /// the same kill.
16617    ///
16618    /// This is the defect reproduction from #109 — a direct-child kill reaches
16619    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
16620    /// supervisor because `spawn_and_mark_running` now always contains on
16621    /// Windows, which is the point: there is no longer a path that spawns
16622    /// uncontained, so the control has to construct one.
16623    ///
16624    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
16625    /// grandchild ever dies here, that test is passing for a reason unrelated to
16626    /// the job object and the containment claim is unproven.
16627    #[test]
16628    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
16629        let dir = ScratchDir::new("teardown-uncontained");
16630        let pid_file = dir.join("grandchild.pid");
16631        let mut child = std::process::Command::new(stub_path())
16632            .env("FAKE_AFT_NEVER_CONNECT", "1")
16633            .env(
16634                "FAKE_AFT_GRANDCHILD_PID_FILE",
16635                pid_file.display().to_string(),
16636            )
16637            .stdin(std::process::Stdio::null())
16638            .stdout(std::process::Stdio::null())
16639            .stderr(std::process::Stdio::null())
16640            .spawn()
16641            .expect("spawn the uncontained fixture");
16642        let grandchild = read_grandchild_pid(&pid_file);
16643
16644        // Exactly what the pre-fix teardown did: kill the direct child.
16645        child.kill().expect("kill the direct child");
16646        let _ = child.wait();
16647
16648        assert!(
16649            subc_jobobject::process_exists(grandchild),
16650            "grandchild {grandchild} died with the direct child, so this control no longer \
16651             distinguishes contained from uncontained teardown and the regression test is \
16652             passing vacuously"
16653        );
16654
16655        // The orphan this control demonstrates is the leak the fix prevents, so
16656        // the control must not leave one behind.
16657        kill_tree(grandchild);
16658    }
16659
16660    /// Crash durability: closing the containment handle reaps the tree with no
16661    /// teardown code running at all.
16662    ///
16663    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
16664    /// call anything — and it is why containment is a kernel property of the
16665    /// handle rather than a step in the drain. Discovered by getting the
16666    /// mutation control wrong: clearing `job` to "disable" containment instead
16667    /// killed the tree, which is the guarantee, not a mistake.
16668    #[tokio::test]
16669    async fn dropping_containment_reaps_the_grandchild() {
16670        let mut fixture = fixture("drop-containment", "tree-drop");
16671        let grandchild = fixture.grandchild;
16672
16673        assert!(subc_jobobject::process_exists(grandchild));
16674
16675        // No `drain` call, no kill: dropping the handle is the entire mechanism.
16676        fixture.child.as_mut().expect("child present").job = None;
16677
16678        assert!(
16679            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16680            "grandchild {grandchild} survived the containment handle closing, so a daemon \
16681             crash would leave the tree behind"
16682        );
16683    }
16684
16685    /// Kill a pid and its tree, then confirm it is gone.
16686    fn kill_tree(pid: u32) {
16687        let _ = std::process::Command::new("taskkill.exe")
16688            .args(["/PID", &pid.to_string(), "/T", "/F"])
16689            .stdin(std::process::Stdio::null())
16690            .stdout(std::process::Stdio::null())
16691            .stderr(std::process::Stdio::null())
16692            .status();
16693        assert!(
16694            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
16695            "could not clean up grandchild {pid}"
16696        );
16697    }
16698}
16699
16700#[cfg(test)]
16701mod privacy_trampoline_configuration_tests {
16702    #[tokio::test]
16703    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16704    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
16705        #[cfg(target_os = "macos")]
16706        {
16707            let supervisor = super::Supervisor::new(
16708                std::sync::Arc::new(crate::Registry::default()),
16709                super::RestartPolicy::default(),
16710            );
16711            let error = supervisor.spawn(spec()).unwrap_err();
16712            assert!(
16713                error
16714                    .to_string()
16715                    .contains("no privacy trampoline configured"),
16716                "{error}"
16717            );
16718        }
16719    }
16720
16721    #[tokio::test]
16722    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16723    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
16724        #[cfg(target_os = "macos")]
16725        {
16726            let supervisor = super::Supervisor::new(
16727                std::sync::Arc::new(crate::Registry::default()),
16728                super::RestartPolicy::default(),
16729            )
16730            .with_privacy_trampoline(std::env::current_exe().unwrap());
16731            let error = supervisor.spawn(spec()).unwrap_err();
16732            assert!(
16733                error
16734                    .to_string()
16735                    .contains("binary does not implement the privacy trampoline protocol"),
16736                "{error}"
16737            );
16738        }
16739    }
16740
16741    #[cfg(target_os = "macos")]
16742    fn spec() -> super::ModuleSpec {
16743        super::ModuleSpec {
16744            module_id: "privacy-configuration".into(),
16745            program: "/bin/sleep".into(),
16746            args: vec!["30".into()],
16747            env: vec![],
16748            reserved: false,
16749            reserved_prefixes: vec![],
16750            protocol: subc_control::ModuleProtocol::None,
16751            overlap: super::ModuleOverlap::Exclusive,
16752        }
16753    }
16754}
16755
16756#[cfg(test)]
16757mod privacy_exec_boundary_tests {
16758    #[cfg(target_os = "macos")]
16759    use super::*;
16760    #[cfg(target_os = "macos")]
16761    use std::{
16762        io::{Read, Write},
16763        net::{TcpListener, TcpStream},
16764    };
16765
16766    /// Unit-test-only pause at the actual early image read, not at a later
16767    /// status read. Production supervisors never inspect this environment key.
16768    #[cfg(target_os = "macos")]
16769    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
16770        if let Some((_, path)) = spec
16771            .env
16772            .iter()
16773            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
16774        {
16775            let mut barrier = TcpStream::connect(path).unwrap();
16776            barrier
16777                .set_read_timeout(Some(Duration::from_secs(30)))
16778                .unwrap();
16779            barrier.write_all(&pid.to_ne_bytes()).unwrap();
16780            let mut release = [0];
16781            barrier.read_exact(&mut release).unwrap();
16782            assert_eq!(&release, b"X");
16783        }
16784    }
16785
16786    #[cfg(target_os = "macos")]
16787    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
16788        ModuleSpec {
16789            module_id: "privacy-boundary".into(),
16790            program: program.into(),
16791            args: args.iter().map(|arg| (*arg).into()).collect(),
16792            env: vec![],
16793            reserved: false,
16794            reserved_prefixes: vec![],
16795            protocol: ModuleProtocol::None,
16796            overlap: ModuleOverlap::Exclusive,
16797        }
16798    }
16799
16800    #[cfg(target_os = "macos")]
16801    async fn accept(listener: TcpListener) -> TcpStream {
16802        // Socket readiness, not elapsed time, establishes both pause points.
16803        let listener = tokio::net::TcpListener::from_std({
16804            listener.set_nonblocking(true).unwrap();
16805            listener
16806        })
16807        .unwrap();
16808        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
16809            .await
16810            .unwrap()
16811            .unwrap();
16812        let stream = stream.into_std().unwrap();
16813        stream.set_nonblocking(false).unwrap();
16814        stream
16815            .set_read_timeout(Some(Duration::from_secs(30)))
16816            .unwrap();
16817        stream
16818    }
16819
16820    #[tokio::test]
16821    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16822    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
16823        #[cfg(target_os = "macos")]
16824        {
16825            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
16826            // Loopback sockets also work when the replay adapter's TMPDIR is
16827            // longer than Darwin's Unix-domain socket path limit.
16828            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16829            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16830            let record = root.join("live-children.json");
16831            let supervisor =
16832                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16833                    .with_live_children_record(&record);
16834            let runtime = supervisor.runtime_config();
16835            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16836            let mut spec = spec("/bin/sleep", &["30"]);
16837            spec.env = vec![
16838                (
16839                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
16840                    exec_listener.local_addr().unwrap().to_string(),
16841                ),
16842                (
16843                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
16844                    sample_listener.local_addr().unwrap().to_string(),
16845                ),
16846            ];
16847            let spawn = tokio::task::spawn_blocking(move || {
16848                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
16849            });
16850            let mut sample = accept(sample_listener).await;
16851            let mut pid = [0; 4];
16852            sample.read_exact(&mut pid).unwrap();
16853            let pid = u32::from_ne_bytes(pid);
16854            let mut exec = accept(exec_listener).await;
16855            let mut ready = [0];
16856            exec.read_exact(&mut ready).unwrap();
16857            assert_eq!(&ready, b"R");
16858            // The early read is guaranteed to see a real, non-null trampoline
16859            // image: the fixture has reached its barrier and cannot exec yet.
16860            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
16861            assert_eq!(
16862                observe_spawned_image(pid).unwrap().executable,
16863                Some(trampoline)
16864            );
16865            sample.write_all(b"X").unwrap();
16866            let mut child = spawn.await.unwrap();
16867            let early = crate::live_children::read_record(&record).unwrap();
16868            assert_eq!(early.len(), 1);
16869            assert_eq!(early[0].pid, pid);
16870            assert_eq!(
16871                early[0].executable, None,
16872                "unconfirmed trampoline image entered the roster"
16873            );
16874            assert!(child.report_ready.get().is_none());
16875            // The barrier's duration is unrelated to the production five-second
16876            // exec budget. Start the test's confirmation budget upon release.
16877            child.privacy_exec.as_mut().unwrap().deadline =
16878                tokio::time::Instant::now() + Duration::from_secs(30);
16879            exec.write_all(b"X").unwrap();
16880            child.confirm_privacy_exec().await;
16881            assert_eq!(child.spawn_failure, None);
16882            assert!(child.report_ready.get().is_some());
16883            let confirmed = crate::live_children::read_record(&record).unwrap();
16884            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
16885            assert_ne!(module, trampoline);
16886            assert_eq!(confirmed[0].executable, Some(module.into()));
16887            child.start_kill().unwrap();
16888            child.wait().await.unwrap();
16889            child.release_roster();
16890        }
16891    }
16892
16893    #[tokio::test]
16894    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16895    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
16896        #[cfg(target_os = "macos")]
16897        {
16898            let registry = Arc::new(Registry::default());
16899            let policy = RestartPolicy::new(0, Duration::ZERO);
16900            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
16901            let runtime = supervisor.runtime_config();
16902            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16903            let spec = spec("/bin/sh", &["-c", "exit 121"]);
16904            // Drive spawn and confirmation separately instead of starting the
16905            // monitor. WNOWAIT observes a real exit without consuming its status,
16906            // so confirmation's first try_wait must take the already-exited arm.
16907            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
16908            let pid = child.pid;
16909            tokio::task::spawn_blocking(move || {
16910                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
16911            })
16912            .await
16913            .unwrap()
16914            .unwrap();
16915            child.privacy_exec.as_mut().unwrap().deadline =
16916                tokio::time::Instant::now() + Duration::from_secs(30);
16917            let status = child.wait().await.unwrap();
16918            assert_eq!(status.code(), Some(121));
16919            assert!(child.privacy_exec.is_none());
16920            assert!(
16921                child.report_ready.get().is_none(),
16922                "an exited module must not publish a live pid"
16923            );
16924            let report = classify_reaped_child_exit(&snapshot, &child, &status);
16925            on_child_exit(
16926                &spec,
16927                policy,
16928                &registry,
16929                &snapshot,
16930                &runtime.terminal_ring,
16931                &runtime.spawn_events,
16932                &runtime.child_roster,
16933                report,
16934                true,
16935            )
16936            .await;
16937            let state = lock_snapshot(&snapshot).unwrap();
16938            assert_eq!(state.state, ModuleState::Failed);
16939            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
16940            assert_eq!(state.reported_pid(), None);
16941            drop(state);
16942            let history = runtime.terminal_ring.lock().unwrap().snapshot();
16943            assert_eq!(history.entries.len(), 1);
16944            let terminal = &history.entries[0];
16945            assert_eq!(terminal.exit_code, Some(121));
16946            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
16947            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
16948            assert_eq!(
16949                terminal.disposition_detail.as_deref(),
16950                Some(policy.budget_exhausted_detail().as_str()),
16951                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
16952            );
16953            assert_eq!(child.spawn_failure, None);
16954            child.release_roster();
16955        }
16956    }
16957}
16958
16959/// The daemon's real spawn path hands a subc-wire child its launch nonce on
16960/// descriptor 3, without an environment copy. The shell records the nonce
16961/// and its environment after exec so these tests observe the real handover.
16962#[cfg(all(test, unix))]
16963mod launch_nonce_descriptor_tests {
16964    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
16965    use crate::stderr_tail::{StderrRing, StderrTailConfig};
16966    use cortexkit_test_support::ScratchDir;
16967    use std::{
16968        path::PathBuf,
16969        sync::{Arc, Mutex},
16970        time::{Duration, Instant},
16971    };
16972
16973    async fn probe(role: super::SpawnRole) {
16974        let scratch = ScratchDir::new("launch-nonce-descriptor");
16975        let fd_copy = scratch.join("from-descriptor");
16976        let env_copy = scratch.join("environment");
16977        let script = format!(
16978            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
16979            fd = fd_copy.display(), env = env_copy.display(),
16980        );
16981        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
16982        let spec = ModuleSpec {
16983            module_id: "nonce-descriptor-probe".to_string(),
16984            program: PathBuf::from("/bin/sh"),
16985            args: vec!["-c".to_string(), script],
16986            env: vec![
16987                xdg("XDG_DATA_HOME"),
16988                xdg("XDG_RUNTIME_DIR"),
16989                xdg("XDG_CONFIG_HOME"),
16990                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
16991            ],
16992            reserved: true,
16993            reserved_prefixes: Vec::new(),
16994            protocol: ModuleProtocol::Subc,
16995            overlap: Default::default(),
16996        };
16997        let handle = SupervisorHandle::new();
16998        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16999        let roster = ChildRoster::default();
17000        #[cfg(target_os = "macos")]
17001        {
17002            let path = super::test_privacy_trampoline();
17003            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
17004        }
17005        let child = super::spawn_child_in_slot(
17006            &spec,
17007            None,
17008            Some(&handle),
17009            &ring,
17010            None,
17011            &roster,
17012            #[cfg(target_os = "linux")]
17013            None,
17014            role,
17015            matches!(role, super::SpawnRole::SwapCandidate),
17016        )
17017        .expect("spawn probe");
17018        let deadline = Instant::now() + Duration::from_secs(10);
17019        while !(fd_copy.exists() && env_copy.exists()) {
17020            assert!(Instant::now() < deadline, "probe never wrote its copies");
17021            tokio::time::sleep(Duration::from_millis(20)).await;
17022        }
17023        let nonce = std::fs::read_to_string(fd_copy).unwrap();
17024        assert!(!nonce.is_empty());
17025        let environment = std::fs::read_to_string(env_copy).unwrap();
17026        assert!(environment
17027            .lines()
17028            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
17029        let copy = environment
17030            .lines()
17031            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
17032        assert_eq!(
17033            copy, None,
17034            "Unix children must never receive the environment nonce"
17035        );
17036        if matches!(role, super::SpawnRole::Plain) {
17037            assert_eq!(
17038                handle.spawn_nonce(&spec.module_id).as_deref(),
17039                Some(nonce.as_str())
17040            );
17041        }
17042        drop(child);
17043    }
17044
17045    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17046    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
17047        probe(super::SpawnRole::Plain).await;
17048    }
17049
17050    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17051    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
17052        probe(super::SpawnRole::SwapCandidate).await;
17053    }
17054}
17055
17056#[cfg(all(test, target_os = "linux"))]
17057mod cgroup_containment_tests {
17058    use super::*;
17059    use cortexkit_test_support::ScratchDir;
17060
17061    fn running(pid: u32) -> bool {
17062        // An orphan can remain a zombie until the container init reaps it.
17063        std::fs::read_to_string(format!("/proc/{pid}/stat"))
17064            .ok()
17065            .and_then(|stat| {
17066                stat.rsplit_once(") ")
17067                    .map(|(_, rest)| rest.starts_with('Z'))
17068            })
17069            .is_some_and(|zombie| !zombie)
17070    }
17071
17072    #[tokio::test]
17073    async fn linux_teardown_reaps_the_grandchild() {
17074        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
17075    }
17076
17077    #[tokio::test]
17078    async fn linux_shutdown_straggler_reaps_the_grandchild() {
17079        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
17080    }
17081
17082    async fn teardown_tree(test_name: &str, shutdown: bool) {
17083        let dir = ScratchDir::new(test_name);
17084        let root = PathBuf::from(format!(
17085            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
17086            std::process::id(),
17087            unix_ms_now()
17088        ));
17089        if let Err(error) = std::fs::create_dir(&root) {
17090            assert!(
17091                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17092                "required cgroup test cannot execute: {error}"
17093            );
17094            eprintln!(
17095                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
17096                root.display()
17097            );
17098            return;
17099        }
17100        let placement = subc_cgroup::prepare_at(&root)
17101            .expect("prepare isolated kernel cgroup")
17102            .expect("isolated cgroup is delegated");
17103        let module_id = "tree-teardown";
17104        let module = placement
17105            .module_path(module_id)
17106            .expect("create isolated module cgroup");
17107        if !module.join("cgroup.kill").exists() {
17108            std::fs::remove_dir(&module).unwrap();
17109            std::fs::remove_dir(root.join("subc-modules")).unwrap();
17110            std::fs::remove_dir(&root).unwrap();
17111            assert!(
17112                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17113                "required cgroup.kill interface unavailable"
17114            );
17115            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
17116            return;
17117        }
17118        let supervisor = Supervisor::new_for_test(
17119            Arc::new(Registry::default()),
17120            RestartPolicy::new(3, Duration::ZERO),
17121        )
17122        .with_cgroup_placement(Some(placement));
17123        let mut runtime = supervisor.runtime_config();
17124        runtime.child_roster = runtime
17125            .child_roster
17126            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
17127        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17128        let pid_file = dir.join("grandchild.pid");
17129        let spec = ModuleSpec {
17130            module_id: module_id.to_string(),
17131            program: PathBuf::from("/bin/sh"),
17132            args: vec![
17133                "-c".into(),
17134                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
17135                "fixture".into(),
17136                pid_file.display().to_string(),
17137            ],
17138            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17139                .into_iter()
17140                .map(|key| (key.to_string(), dir.display().to_string()))
17141                .collect(),
17142            reserved: false,
17143            reserved_prefixes: Vec::new(),
17144            protocol: ModuleProtocol::None,
17145            overlap: Default::default(),
17146        };
17147        let child =
17148            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
17149        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
17150        let grandchild: u32 = loop {
17151            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
17152                if let Ok(pid) = contents.trim().parse() {
17153                    break pid;
17154                }
17155            }
17156            assert!(
17157                tokio::time::Instant::now() < deadline,
17158                "grandchild pid was not recorded"
17159            );
17160            tokio::time::sleep(Duration::from_millis(10)).await;
17161        };
17162        assert!(
17163            running(grandchild),
17164            "grandchild must be alive before teardown"
17165        );
17166        if shutdown {
17167            let mut child = child;
17168            crate::child_roster::end_children_for_daemon_shutdown(
17169                &runtime.child_roster,
17170                false,
17171                std::future::pending(),
17172            )
17173            .await;
17174            child.wait().await.expect("reap shutdown straggler");
17175        } else {
17176            drain_child_to_state(
17177                module_id,
17178                ModuleProtocol::None,
17179                StopNotice::NotSent,
17180                &Registry::default(),
17181                None,
17182                &snapshot,
17183                &runtime.terminal_ring,
17184                &SpawnEventFeed::default(),
17185                child,
17186                Duration::from_millis(100),
17187                ModuleState::Stopped,
17188                Some(false),
17189                &FailureContext::operator(),
17190            )
17191            .await
17192            .expect("real supervisor teardown");
17193        }
17194        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
17195        while running(grandchild) && tokio::time::Instant::now() < deadline {
17196            tokio::time::sleep(Duration::from_millis(10)).await;
17197        }
17198        let survived = running(grandchild);
17199        // Kill a surviving grandchild so a failed test does not leave it behind.
17200        if survived {
17201            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
17202            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
17203            tokio::time::sleep(Duration::from_millis(100)).await;
17204        }
17205        if module.exists() {
17206            std::fs::remove_dir(&module).expect("remove empty module cgroup");
17207        }
17208        std::fs::remove_dir(root.join("subc-modules")).unwrap();
17209        std::fs::remove_dir(&root).unwrap();
17210        assert!(
17211            !survived,
17212            "grandchild {grandchild} outlived module teardown"
17213        );
17214        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
17215    }
17216}
17217
17218/// A slow live-children record write on the restart path must not stop the
17219/// daemon from answering the replacement module's authentication handshake.
17220#[cfg(all(test, unix))]
17221mod record_write_stall_tests {
17222    use super::*;
17223    use std::sync::atomic::{AtomicBool, Ordering};
17224    use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
17225
17226    /// `subc-client-rs` gives the whole handshake this long before the module
17227    /// exits with `Timeout { stage: ServerProof }`.
17228    const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
17229    /// How long the replacement's admission write is held: longer than the
17230    /// module's deadline, standing in for the delay a busy disk can add
17231    /// across the two syncs in that write.
17232    const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
17233
17234    /// A module that runs `/bin/sleep` directly, not through `/bin/sh -c`.
17235    /// On macOS every module is launched through the privacy trampoline, and
17236    /// a module that is not a `#!` script is admitted only if its running
17237    /// image is the configured program. With `/bin/sh -c "exec sleep 600"`
17238    /// the shell soon replaces itself with `/bin/sleep`, which does not match
17239    /// the configured `/bin/sh`. The image is read only after the spawn's
17240    /// live-children record write, and here that write lasted long enough for
17241    /// the shell to have exec'd every time, so each process was refused and
17242    /// killed right after spawn, and the supervisor crash-looped it with
17243    /// backoff until its restart budget ran out. The restart this test
17244    /// measures would then never produce a module at all.
17245    fn sleeping_spec() -> ModuleSpec {
17246        ModuleSpec {
17247            module_id: "record-write-stall".into(),
17248            program: "/bin/sleep".into(),
17249            args: vec!["600".into()],
17250            env: vec![],
17251            reserved: false,
17252            reserved_prefixes: vec![],
17253            protocol: ModuleProtocol::None,
17254            overlap: Default::default(),
17255        }
17256    }
17257
17258    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17259    async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
17260        let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
17261        let supervisor =
17262            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17263                .with_live_children_record(home.join("live-children.json"));
17264        let module = supervisor.spawn(sleeping_spec()).unwrap();
17265        // `reported_pid` names a process only once it is confirmed as the
17266        // module (on macOS, once the trampoline's image check has passed), so
17267        // a process the daemon is about to refuse and kill never counts as
17268        // the running module, here or in the final wait below.
17269        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
17270            loop {
17271                if let Some(pid) = lock_snapshot(&module.inner.snapshot)
17272                    .unwrap()
17273                    .reported_pid()
17274                {
17275                    return pid;
17276                }
17277                tokio::time::sleep(Duration::from_millis(10)).await;
17278            }
17279        })
17280        .await
17281        .expect("the first process was admitted as the module");
17282
17283        // The daemon's listener, served by this runtime's workers like the
17284        // supervisor's own tasks.
17285        let key = vec![0x42; 32];
17286        let daemon_id = [0x24; 16];
17287        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
17288        let port = listener.local_addr().unwrap().port();
17289        let server = tokio::spawn(crate::server::serve_listener(
17290            listener,
17291            Arc::new(crate::router::Router::with_default_self_handler()),
17292            crate::server::ServerAuth::with_limits(
17293                key.clone(),
17294                daemon_id,
17295                "test",
17296                crate::server::DEFAULT_AUTH_DEADLINE,
17297                4,
17298            ),
17299        ));
17300
17301        // Hold the write that admits the replacement, once, and say when it
17302        // has started.
17303        let (held_tx, held_rx) = std::sync::mpsc::channel();
17304        let armed = AtomicBool::new(true);
17305        supervisor
17306            .child_roster
17307            .set_record_write_hook_for_test(move |live| {
17308                let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
17309                if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
17310                    let _ = held_tx.send(());
17311                    std::thread::sleep(HELD_RECORD_WRITE);
17312                }
17313            });
17314
17315        // The replacement module, as a separate process would be: its own
17316        // thread and runtime, so only the daemon's runtime can be stalled.
17317        let module_process = std::thread::spawn(move || {
17318            held_rx
17319                .recv_timeout(Duration::from_secs(30))
17320                .expect("the restart reached the replacement's record write");
17321            let conn = ConnectionInfo {
17322                schema: SCHEMA_VERSION,
17323                wire_version: None,
17324                endpoints: vec![Endpoint {
17325                    host: "127.0.0.1".to_owned(),
17326                    port,
17327                }],
17328                key,
17329                daemon_id,
17330                pid: std::process::id(),
17331                daemon_ver: "test".to_owned(),
17332            };
17333            tokio::runtime::Builder::new_current_thread()
17334                .enable_all()
17335                .build()
17336                .unwrap()
17337                .block_on(async {
17338                    let started = Instant::now();
17339                    let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
17340                        .await
17341                        .unwrap();
17342                    let result =
17343                        authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
17344                    (result, started.elapsed())
17345                })
17346        });
17347
17348        module.restart(Some(0)).await.unwrap();
17349        let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
17350            .await
17351            .unwrap();
17352        assert!(
17353            auth.is_ok(),
17354            "the daemon must answer a module's handshake while a restart waits on the \
17355             live-children record write; got {auth:?} after {elapsed:?}"
17356        );
17357
17358        tokio::time::timeout(Duration::from_secs(20), async {
17359            while lock_snapshot(&module.inner.snapshot)
17360                .unwrap()
17361                .reported_pid()
17362                .is_none_or(|pid| pid == old_pid)
17363            {
17364                tokio::time::sleep(Duration::from_millis(10)).await;
17365            }
17366        })
17367        .await
17368        .expect("the restart published its replacement");
17369        module.stop().await.unwrap();
17370        server.abort();
17371    }
17372}