Skip to main content

subc_daemon/
supervise.rs

1use std::{
2    collections::{HashMap, HashSet, VecDeque},
3    error::Error,
4    fmt, io,
5    path::PathBuf,
6    process::{ExitStatus, Stdio},
7    sync::{Arc, Mutex, OnceLock},
8    time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use subc_control::{
14    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
15    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
16};
17use subc_protocol::{
18    manifest::{SelfSignalKind, SignalAnchor},
19    session::{
20        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
21        MODULE_CONTROL_OP_HEALTH_CHECK,
22    },
23    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
24};
25use tokio::{
26    process::{Child, Command},
27    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
28    task::JoinHandle,
29    time::{sleep, sleep_until, timeout, timeout_at, Instant},
30};
31use tracing::{debug, error, info, warn};
32
33use crate::{
34    child_roster::ChildRoster,
35    daemon_config::{
36        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
37    },
38    forwarding::{
39        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
40        ModuleDrainTarget, PendingModuleControlRpc,
41    },
42    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
43    registry::{ConnectionId, RegistrationEndReason, RegistryError},
44    stderr_tail::{
45        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
46        StderrTailSnapshot,
47    },
48    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
49    Frame, FrameSink, Registry,
50};
51
52#[cfg(not(windows))]
53use crate::provenance::spawned_file_identity;
54
55#[path = "supervise_swap.rs"]
56mod swap;
57
58#[path = "supervise_activity.rs"]
59mod activity;
60
61/// Command-line flag used by supervised modules to find subc.
62///
63/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
64/// The path points at the TCP+key connection file; it is not an ambient signal and
65/// is never inherited by standalone children.
66pub const SUBC_ARG: &str = "--subc";
67
68const DEFAULT_MAX_RESTARTS: u32 = 3;
69const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
70const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
71/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
72/// to contain a real crash loop (which respawns in seconds) and short enough
73/// that unrelated crashes hours apart never accumulate into a permanent stop.
74const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
75/// How long a drain waits for already-dispatched requests to finalize before
76/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
77/// not RPC-scale: the original 2s value silently cut nearly every real tool
78/// call at the fence, making the wait-for-finalize design decorative for the
79/// workloads it existed for. Quiescence short-circuits, so an idle module
80/// restarts immediately regardless of this value; the budget is spent only
81/// when a genuine in-flight request is worth finishing. Per-module override:
82/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
83/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
84/// where a stuck request will never settle).
85pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
86const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
87const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
88/// How long a restart waits for the exited process's output readers.
89///
90/// The restart does not depend on either output reader finishing. A stderr
91/// reader still running at this bound is left running, and its later output goes
92/// into the exited process's own section of the stderr ring (see
93/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
94/// bound no longer decides whether a crash's last lines are kept: under load
95/// the reader may simply not have been scheduled yet, and cutting it there
96/// discarded exactly the lines that explained the crash.
97///
98/// What the bound still decides is when the tail starts reporting
99/// `Incomplete`: a pipe open past it usually means a descendant of the exited
100/// process still holds it, and the tail cannot claim to be whole until that
101/// pipe closes. That is also the only case in which the wait costs the restart
102/// anything, because a pipe with no other holder reaches EOF when the process
103/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
104/// `Captured` at EOF with nothing lost.
105///
106/// The stdout reader carries no ring, only the capture file. It also continues
107/// to EOF after this bound: child exit closes the writer but does not mean the
108/// reader has consumed all buffered bytes. Late output from either pipe can
109/// therefore reach the shared capture file after the successor starts.
110const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
111/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
112pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
113const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
114/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
115/// dropped because the subscriber stopped draining its frames. The detail's
116/// `first_undelivered_cursor` names the first event it did not receive; the
117/// client resubscribes from the last cursor it did receive.
118pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
119
120#[cfg(target_os = "macos")]
121struct PrivacyExec {
122    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
123    deadline: tokio::time::Instant,
124    expected: Option<subc_os::FileIdentity>,
125    trampoline: Option<subc_os::FileIdentity>,
126    script: bool,
127    module_id: String,
128    #[cfg(test)]
129    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
130}
131
132#[cfg(all(test, target_os = "macos"))]
133pub(crate) fn test_privacy_trampoline() -> PathBuf {
134    // Cargo's unit-test executable lives in <profile>/deps; its fixture bin
135    // lives beside that directory. This honors custom CARGO_TARGET_DIR too.
136    let path = std::env::current_exe()
137        .unwrap()
138        .parent()
139        .unwrap()
140        .parent()
141        .unwrap()
142        .join("privacy-trampoline-fixture");
143    // Without the fixture every macOS spawn is refused, and the tests that
144    // spawn fail later as a module in state Failed, which names the wrong
145    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
146    assert!(
147        path.exists(),
148        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
149         --bins --features test-support` or `cargo test -p subc-daemon` first",
150        path.display()
151    );
152    path
153}
154
155#[cfg(target_os = "macos")]
156fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
157    use std::io::Read;
158    let mut probe = std::process::Command::new(path)
159        .args(["__disclaim-exec", "--probe"])
160        .stdin(Stdio::null())
161        .stdout(Stdio::piped())
162        .stderr(Stdio::piped())
163        .spawn()
164        .map_err(|error| {
165            format!(
166                "privacy trampoline probe failed for {}: {error}",
167                path.display()
168            )
169        })?;
170    let deadline = std::time::Instant::now() + Duration::from_secs(5);
171    let status = loop {
172        match probe.try_wait() {
173            Ok(Some(status)) => break status,
174            Ok(None) if std::time::Instant::now() < deadline => {
175                std::thread::sleep(Duration::from_millis(5))
176            }
177            result => {
178                let _ = probe.kill();
179                let _ = probe.wait();
180                return Err(format!(
181                    "privacy trampoline probe failed or timed out for {}: {result:?}",
182                    path.display()
183                ));
184            }
185        }
186    };
187    let mut answer = String::new();
188    if let Some(stdout) = probe.stdout.take() {
189        let _ = stdout.take(256).read_to_string(&mut answer);
190    }
191    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
192        return Ok(());
193    }
194    let mut diagnostic = String::new();
195    if let Some(stderr) = probe.stderr.take() {
196        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
197    }
198    let cause = diagnostic
199        .trim()
200        .strip_prefix("ck-subc: own privacy identity refused: ")
201        .unwrap_or("binary does not implement the privacy trampoline protocol");
202    Err(format!(
203        "{cause}: probe of {} exited {status}",
204        path.display()
205    ))
206}
207
208#[cfg(target_os = "macos")]
209fn privacy_command(
210    spec: &ModuleSpec,
211    roster: &ChildRoster,
212) -> Result<
213    (
214        Command,
215        Option<PrivacyExec>,
216        subc_os::privacy_identity::ExecAcknowledgement,
217    ),
218    SuperviseError,
219> {
220    let failure = |cause: String| {
221        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
222        SuperviseError::Spawn {
223            program: spec.program.clone(),
224            source: io::Error::other(cause),
225            cgroup_path: None,
226        }
227    };
228    let trampoline = roster.privacy_trampoline().map_err(failure)?;
229    // Resolve PATH with the same environment the Command will receive. For
230    // scripts retain the existing orphan-identity rule: the kernel chooses
231    // the interpreter, and its observed image is the one recorded. Do not
232    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
233    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
234        let path = spec
235            .env
236            .iter()
237            .find(|(key, _)| key == "PATH")
238            .map(|(_, value)| std::ffi::OsString::from(value))
239            .or_else(|| std::env::var_os("PATH"))
240            .unwrap_or_else(|| "/usr/bin:/bin".into());
241        std::env::split_paths(&path)
242            .map(|dir| dir.join(&spec.program))
243            .find(|path| path.is_file())
244            .unwrap_or_else(|| spec.program.clone())
245    } else {
246        spec.program.clone()
247    };
248    let expected = subc_os::file_identity(&program);
249    let trampoline_image = subc_os::file_identity(&trampoline);
250    let script = {
251        use std::io::Read;
252        let mut prefix = [0u8; 2];
253        std::fs::File::open(&program)
254            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
255    };
256    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
257        return Err(failure(
258            "privacy identity module executable is missing or is the trampoline itself".to_string(),
259        ));
260    }
261    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
262        .map_err(|error| failure(error.to_string()))?;
263    let reader =
264        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
265    let mut command = Command::new(&trampoline);
266    command
267        .arg("__disclaim-exec")
268        .arg(ack.fd().to_string())
269        .arg(&program);
270    ack.install(command.as_std_mut());
271    Ok((
272        command,
273        Some(PrivacyExec {
274            reader,
275            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
276            expected,
277            trampoline: trampoline_image,
278            script,
279            module_id: spec.module_id.clone(),
280            #[cfg(test)]
281            confirmation_pause: spec
282                .env
283                .iter()
284                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
285                .map(|(_, address)| (address.clone(), None)),
286        }),
287        ack,
288    ))
289}
290
291struct SupervisedChild {
292    child: Child,
293    exec: crate::activity::ExecOwner,
294    #[cfg(windows)]
295    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
296    #[cfg(target_os = "macos")]
297    privacy_exec: Option<PrivacyExec>,
298    /// Set once this launch's exec acknowledgement confirms the module image.
299    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
300    /// `subc_os::privacy_identity`), which then replaces itself with the
301    /// module. The supervisor owns and can kill that pid from spawn, but
302    /// status readers report it only after this latch is set, so nothing
303    /// reports the trampoline's image as the module's.
304    #[cfg(target_os = "macos")]
305    report_ready: Arc<OnceLock<()>>,
306    /// Refusal before the module image was accepted, retained for terminal records.
307    spawn_failure: Option<String>,
308    /// The protocol this process was launched with. A reload can store a new
309    /// launch spec with a different protocol, but that takes effect only at the
310    /// next spawn, so this process keeps being handled by the protocol it
311    /// actually speaks.
312    protocol: ModuleProtocol,
313    /// This process's cgroup name: a bounded module/slot label followed by a
314    /// spawn suffix unique to this process (when cgroup placement is on). A
315    /// retired process in a slot may still be draining when a later one is
316    /// spawned into that slot, so the suffix keeps the later process out of
317    /// the retired one's cgroup, which is the domain a kill applies to.
318    #[cfg(target_os = "linux")]
319    module_id: String,
320    #[cfg(target_os = "linux")]
321    cgroup_placement: Option<subc_cgroup::Placement>,
322    /// The job that contains this child and every process it spawns (issue #109).
323    ///
324    /// Dropping this handle is what reaps a surviving tree when no supervisor
325    /// code runs — a daemon crash — because the job carries
326    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
327    ///
328    /// That limit is not crash-only, and the difference is worth knowing: a
329    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
330    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
331    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
332    /// module at once. Before this change they survived that, saw EOF on the
333    /// control socket, and ran their own teardown; Unix keeps that path
334    /// deliberately, so a module can seal a WAL or close a capture rather than
335    /// be killed mid-write. So this trades graceful teardown on every Windows
336    /// daemon stop for containment on a crash, which is the right way round
337    /// today: orphaned GPU workers are a reported, recurring problem, and the
338    /// modules that write most heavily do not run on Windows.
339    ///
340    /// The fix is a real Windows stop path — the daemon draining before it
341    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
342    /// reaches only what the drain left behind, which is what it should reach.
343    #[cfg(windows)]
344    job: Option<subc_jobobject::JobObject>,
345    stdout_pump: Option<JoinHandle<()>>,
346    stderr_pump: Option<StderrPump>,
347    stderr_ring: Arc<Mutex<StderrRing>>,
348    spawned_at_ms: u64,
349    spawned_from: PathBuf,
350    spawned_file_identity: Option<SpawnedFileIdentity>,
351    process_start_time: Option<u64>,
352    process_identity: Option<ProcessIdentity>,
353    pid: u32,
354    /// This process's entry in the daemon's child roster, released when the
355    /// process is reaped or this handle is dropped.
356    roster_guard: Option<crate::child_roster::RosterGuard>,
357}
358
359impl SupervisedChild {
360    fn id(&self) -> Option<u32> {
361        Some(self.pid)
362    }
363
364    fn process_identity(&self) -> Option<ProcessIdentity> {
365        self.process_identity
366    }
367
368    async fn wait(&mut self) -> io::Result<ExitStatus> {
369        #[cfg(target_os = "macos")]
370        self.confirm_privacy_exec().await;
371        // The roster entry is NOT released here. A daemon shutdown waits for the
372        // roster to empty and then exits the process, so releasing at the reap
373        // let it exit before the exit handler wrote this child's terminal record
374        // (the stderr drain and snapshot update sit in between), and the
375        // shutdown's own `daemon_shutdown` record was intermittently lost. The
376        // caller releases it after recording the exit (`release_roster`), and
377        // dropping the handle releases it too.
378        let result = self.child.wait().await;
379        #[cfg(target_os = "linux")]
380        if result.is_ok() {
381            if let Some(placement) = self.cgroup_placement.as_ref() {
382                cleanup_reaped_cgroup(placement, &self.module_id).await;
383                // Keep ownership while awaiting kernel population changes: a
384                // drain timeout may cancel this wait and then escalate/reap.
385                self.cgroup_placement = None;
386            }
387        }
388        result
389    }
390
391    #[cfg(target_os = "macos")]
392    async fn confirm_privacy_exec(&mut self) {
393        let Some(pending) = &mut self.privacy_exec else {
394            return;
395        };
396        let result = tokio::time::timeout_at(pending.deadline, async {
397            let mut record = Vec::new();
398            loop {
399                let mut ready = pending.reader.readable().await?;
400                let read = ready.try_io(|reader| {
401                    use std::io::Read;
402                    let mut reader = reader.get_ref();
403                    let mut buffer = [0u8; 256];
404                    reader.read(&mut buffer).map(|count| (count, buffer))
405                });
406                match read {
407                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
408                    Ok(Ok((count, buffer))) => {
409                        if record.len() + count > 1024 {
410                            return Err(io::Error::other(
411                                "privacy exec refusal record is too long",
412                            ));
413                        }
414                        record.extend_from_slice(&buffer[..count]);
415                    }
416                    Ok(Err(error)) => return Err(error),
417                    Err(_) => continue,
418                }
419            }
420        })
421        .await;
422        #[cfg(test)]
423        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
424            use tokio::io::{AsyncReadExt, AsyncWriteExt};
425            if stream.is_none() {
426                *stream = Some(
427                    tokio::net::TcpStream::connect(address.as_str())
428                        .await
429                        .unwrap(),
430                );
431                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
432                stream
433                    .as_mut()
434                    .unwrap()
435                    .write_all(&self.pid.to_ne_bytes())
436                    .await
437                    .unwrap();
438            }
439            let mut reply = [0];
440            stream
441                .as_mut()
442                .unwrap()
443                .read_exact(&mut reply)
444                .await
445                .unwrap();
446            pending.confirmation_pause = None;
447            if &reply == b"F" {
448                Ok(Err(io::Error::other("injected late exec refusal")))
449            } else {
450                assert_eq!(&reply, b"X");
451                result
452            }
453        } else {
454            result
455        };
456        // Keep the reader in self across await: select cancellation must not
457        // discard the handshake or reset its original five-second deadline.
458        let pending = self.privacy_exec.as_ref().expect("pending exec");
459        let cause = match result {
460            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
461            Ok(Err(error)) => Some(format!(
462                "privacy identity exec acknowledgement failed: {error}"
463            )),
464            Ok(Ok(record)) if !record.is_empty() => Some(
465                std::str::from_utf8(&record)
466                    .ok()
467                    .and_then(|record| {
468                        record
469                            .trim()
470                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
471                    })
472                    .filter(|cause| !cause.is_empty())
473                    .unwrap_or("invalid privacy exec refusal record")
474                    .to_string(),
475            ),
476            Ok(Ok(_)) => match self.child.try_wait() {
477                // Empty EOF is the exec acknowledgement. A real module may exit
478                // immediately, including with a reserved trampoline status; no
479                // image is admitted, and its ordinary exit contract stays intact.
480                Ok(Some(_status)) => None,
481                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
482                Ok(None) => {
483                    let image = observe_spawned_image(self.pid);
484                    if let Some(image) = image.filter(|image| {
485                        image.executable.is_some()
486                            && image.executable != pending.trampoline
487                            && (image.executable == pending.expected || pending.script)
488                    }) {
489                        if let Err(error) = self
490                            .roster_guard
491                            .as_ref()
492                            .map(|guard| guard.confirm_executable(image))
493                            .transpose()
494                        {
495                            Some(format!(
496                                "could not durably record confirmed module image: {error}"
497                            ))
498                        } else {
499                            let _ = self.report_ready.set(());
500                            info!(module_id = %pending.module_id, pid = self.pid,
501                                "module spawned with own privacy identity (responsibility disclaimed)");
502                            None
503                        }
504                    } else if image.is_none()
505                        || image.is_some_and(|image| image.executable.is_none())
506                    {
507                        // A process can exit between try_wait and the kernel
508                        // image read. Empty EOF already acknowledged exec, so
509                        // preserve that module's ordinary exit rather than
510                        // mislabel a disappearing image as trampoline refusal.
511                        // Keep pending in self across await so cancellation does
512                        // not discard validation or reset its original deadline.
513                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
514                            Ok(Ok(_status)) => None,
515                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
516                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
517                        }
518                    } else {
519                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
520                    }
521                }
522            },
523        };
524        let pending = self.privacy_exec.take().expect("pending exec");
525        // Exec confirmation describes launch success, not roster retirement.
526        // Keep even a reaped child recorded until its terminal history exists.
527        self.exec.0.finish(if cause.is_some() {
528            crate::activity::ExecState::Refused
529        } else {
530            crate::activity::ExecState::CommittedExec
531        });
532        if let Some(cause) = cause {
533            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
534            self.spawn_failure = Some(cause);
535            // No image is admitted on failure. Reach the entire fresh process
536            // group, including a module which spawned a helper before refusal.
537            if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
538                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
539            }
540            let _ = self.child.start_kill();
541        }
542    }
543
544    /// Releases this child's daemon-shutdown roster entry once its exit has
545    /// been recorded. The pid is already reaped and free for reuse, so the
546    /// entry must not outlive the record any longer than that.
547    fn release_roster(&mut self) {
548        self.roster_guard = None;
549    }
550
551    /// Kill the child and, where containment is available, its process tree.
552    ///
553    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
554    /// helper process leaked the helper — the Synapse embedding module's CUDA
555    /// worker holds the GPU allocation, so the leak cost VRAM until the next
556    /// restart of something else. Terminating the job reaches grandchildren that
557    /// a tree walk cannot, including one whose parent has already exited and
558    /// been reparented away.
559    ///
560    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
561    /// direct-child kill still decides the outcome, so containment can never
562    /// change whether a module is reported as stopped.
563    fn start_kill(&mut self) -> io::Result<()> {
564        #[cfg(windows)]
565        if let Some(job) = &self.job {
566            if let Err(error) = job.terminate() {
567                debug!(
568                    error = %error,
569                    "job termination failed; the direct-child kill still owns the outcome"
570                );
571            }
572        }
573        #[cfg(target_os = "linux")]
574        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
575        self.child.start_kill()
576    }
577
578    async fn drain_stderr(&mut self, module_id: &str) {
579        if let Some(pump) = self.stdout_pump.take() {
580            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
581        }
582
583        let Some(pump) = self.stderr_pump.take() else {
584            return;
585        };
586        settle_stderr_pump(
587            module_id,
588            &self.stderr_ring,
589            pump,
590            STDERR_PUMP_DRAIN_TIMEOUT,
591        )
592        .await;
593    }
594}
595
596/// Bound the restart's wait, not the lifetime of the exited child's stdout
597/// reader. Slow pipe reads or capture writes must not turn the bound into lost
598/// output; dropping the handle lets the reader finish at EOF, like stderr.
599async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
600    match timeout(bound, &mut pump).await {
601        Ok(Ok(())) => {}
602        Ok(Err(error)) => {
603            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
604        }
605        Err(_) => {
606            drop(pump);
607            warn!(
608                module_id,
609                waited = ?bound,
610                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
611            );
612        }
613    }
614}
615
616/// The reader task for one process's stderr, with the ring generation its
617/// lines are attributed to.
618struct StderrPump {
619    task: JoinHandle<()>,
620    generation: u64,
621}
622
623/// Retire an exited process's stderr reader and wait up to `bound` for it to
624/// reach EOF. A reader still running at the bound is detached, not stopped: it
625/// keeps filling the exited process's section of the ring until its pipe
626/// closes, and the tail reads `Incomplete` until then. See
627/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
628async fn settle_stderr_pump(
629    module_id: &str,
630    ring: &Arc<Mutex<StderrRing>>,
631    pump: StderrPump,
632    bound: Duration,
633) {
634    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
635    let StderrPump {
636        mut task,
637        generation,
638    } = pump;
639    lock().retire_pump(generation);
640    match timeout(bound, &mut task).await {
641        Ok(Ok(())) => {}
642        Ok(Err(err)) => {
643            let mut ring = lock();
644            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
645            ring.finish_pump(generation);
646            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
647        }
648        Err(_) => {
649            // Dropping the handle detaches the task; it ends at EOF on its pipe.
650            drop(task);
651            lock().mark_pump_late(
652                generation,
653                format!(
654                    "stderr of the exited process had not reached EOF {bound:?} after it was \
655                     retired (a descendant may still hold the pipe open); lines it still \
656                     writes are kept in that process's section"
657                ),
658            );
659            warn!(
660                module_id,
661                waited = ?bound,
662                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
663            );
664        }
665    }
666}
667
668fn registration_release_events() -> &'static watch::Sender<u64> {
669    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
670    EVENTS.get_or_init(|| {
671        let (sender, _receiver) = watch::channel(0);
672        sender
673    })
674}
675
676pub(crate) fn notify_registration_release() {
677    let events = registration_release_events();
678    let next_generation = (*events.borrow()).wrapping_add(1);
679    events.send_replace(next_generation);
680}
681
682/// How to launch one singleton module process.
683#[derive(Debug, Clone, PartialEq, Eq)]
684pub struct ModuleSpec {
685    pub module_id: String,
686    pub program: PathBuf,
687    pub args: Vec<String>,
688    pub env: Vec<(String, String)>,
689    /// When true this is a reserved module: each spawn gets a fresh one-time launch
690    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
691    /// process can register this module_id (a security-boundary module like the
692    /// credential vault must not be impersonable while it is down/restarting).
693    pub reserved: bool,
694    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
695    /// Prefixes come from daemon config and must end in `:` before they reach the
696    /// supervisor; the owner module's current spawn nonce authorizes claims under
697    /// each prefix.
698    pub reserved_prefixes: Vec<String>,
699    /// The wire protocol this module speaks, as DECLARED in daemon config.
700    ///
701    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
702    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
703    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
704    /// and NO launch nonce, and a clean exit the daemon did not request is
705    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
706    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
707    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
708    /// because a process ignores an environment variable it does not read.
709    ///
710    /// The argument is the part that cannot be "harmless to a process that
711    /// ignores it": a stock binary exits on an unknown flag before it listens
712    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
713    /// first conformance run against this mode found it. The nonce is withheld
714    /// because a process that will never present it gains nothing from holding
715    /// it, and a secret in the environment of a process that does not need it is
716    /// a leak surface for no benefit.
717    pub protocol: ModuleProtocol,
718    /// Whether two processes of this module may run at once, which is what a
719    /// blue/green swap does for the length of its overlap. Declared in daemon
720    /// config because the daemon must be able to answer it while the module is
721    /// down, and so a module cannot talk itself into it after registering.
722    pub overlap: ModuleOverlap,
723}
724
725/// Whether a module tolerates a second process of itself running alongside.
726///
727/// Most modules are single-writer on their store (a WAL, a capture log, a
728/// resident index behind a writer barrier), and two processes on one store
729/// corrupt it. So a swap, which overlaps the old and new process by design,
730/// is refused unless the module's config opts in with `overlap: "safe"`.
731#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
732pub enum ModuleOverlap {
733    /// Never run two processes of this module at once. The default.
734    #[default]
735    Exclusive,
736    /// The module has said a second process of itself is harmless for the
737    /// length of a swap.
738    ///
739    /// Declare it only if a second instance can run for a few seconds without
740    /// touching ANY single-writer store: every database, WAL, index, projector
741    /// and scheduled job the module owns. A lease on part of that state is not
742    /// enough. broca's session lease guards WAL appends while its run index, its
743    /// store projector and its archive fold timer (which unlinks live WAL files)
744    /// stay single-writer, so broca is exclusive despite holding a lease. The
745    /// refusal only fires after this has been decided, so the decision is the
746    /// check.
747    Safe,
748}
749
750impl ModuleOverlap {
751    pub fn as_str(self) -> &'static str {
752        match self {
753            Self::Exclusive => "exclusive",
754            Self::Safe => "safe",
755        }
756    }
757}
758
759/// Environment variable telling a spawned module which case it was started
760/// for, before it sends HELLO. Only a swap candidate carries it, as
761/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
762///
763/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
764/// longer because nobody waits on it, while a plain restart must flip ready
765/// quickly because callers see `module_warming` until it does. Absence means
766/// plain restart, the safe reading. The daemon trusts nothing about it; the
767/// candidate is proven by its launch nonce at HELLO.
768pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
769/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
770pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
771/// How long a swap waits for its candidate to register and declare itself
772/// ready when the operator does not say. A module warming as a swap candidate
773/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
774/// daemon allows that plus time to start the process and send HELLO.
775pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
776
777/// Bounded restart policy for crash exits.
778///
779/// `max_restarts` is the number of replacement processes allowed after the
780/// initial spawn WITHIN `window`. After that many crash restarts inside one
781/// window the module enters [`ModuleState::Failed`] and the supervisor stops
782/// the crash loop.
783///
784/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
785/// and that only survived because crashes were rare: a module that crashed
786/// three times across a week was disabled forever by crashes that had nothing
787/// to do with each other. That stopped being survivable once modules began
788/// exiting non-zero whenever the daemon's connection to them drops, because
789/// then every daemon-side connection drop spends a unit of the same budget and
790/// one flappy hour permanently stops a healthy module. Restarts older than
791/// `window` release their slot, so a module that crashed twice yesterday has a
792/// full budget today, while a genuine crash loop -- which is fast by
793/// definition -- still reaches the cap and stops.
794#[derive(Debug, Clone, Copy, PartialEq, Eq)]
795pub struct RestartPolicy {
796    pub max_restarts: u32,
797    /// Base delay before a crash replacement. The actual delay escalates with
798    /// the number of recent crash replacements and is capped by `max_backoff`.
799    pub backoff: Duration,
800    /// Maximum delay before a crash replacement.
801    pub max_backoff: Duration,
802    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
803    /// budget effectively infinite (nothing is ever in-window), which is why
804    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
805    pub window: Duration,
806}
807
808impl RestartPolicy {
809    /// A policy with the default crash window. Callers that care about the
810    /// window say so with [`Self::with_window`]; the ones that do not are
811    /// asking for the standard rate limit, not for no limit.
812    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
813        Self {
814            max_restarts,
815            backoff,
816            max_backoff: DEFAULT_MAX_BACKOFF,
817            window: DEFAULT_RESTART_WINDOW,
818        }
819    }
820
821    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
822        self.max_backoff = max_backoff;
823        self
824    }
825
826    pub fn with_window(mut self, window: Duration) -> Self {
827        self.window = window;
828        self
829    }
830
831    /// Calculate the capped exponential delay for the next crash replacement.
832    /// `restart_in_window` is zero for the first replacement after an operator
833    /// action (restart, reload, re-enable) cleared the crash ring, or after all
834    /// older crash replacements have aged out of the window.
835    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
836        if self.backoff.is_zero() || self.max_backoff.is_zero() {
837            return Duration::ZERO;
838        }
839
840        let mut delay = self.backoff;
841        for _ in 0..restart_in_window {
842            if delay >= self.max_backoff {
843                return self.max_backoff;
844            }
845            delay = delay
846                .checked_mul(10)
847                .unwrap_or(self.max_backoff)
848                .min(self.max_backoff);
849        }
850        delay.min(self.max_backoff)
851    }
852
853    /// The one sentence that explains a budget-exhausted stop, used for both the
854    /// log line and the terminal record so the two cannot drift. It names the
855    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
856    /// exactly what this budget is not.
857    fn budget_exhausted_detail(&self) -> String {
858        format!(
859            "crash budget exhausted: max_restarts={} within window_secs={}",
860            self.max_restarts,
861            self.window.as_secs()
862        )
863    }
864}
865
866impl Default for RestartPolicy {
867    fn default() -> Self {
868        Self {
869            max_restarts: DEFAULT_MAX_RESTARTS,
870            backoff: DEFAULT_BACKOFF,
871            max_backoff: DEFAULT_MAX_BACKOFF,
872            window: DEFAULT_RESTART_WINDOW,
873        }
874    }
875}
876
877#[derive(Debug, Clone, Copy, PartialEq, Eq)]
878struct CrashRestartSchedule {
879    restart_in_window: u32,
880    delay: Duration,
881}
882
883/// Whether the daemon itself will bring this module back after the exit being
884/// handled: it is enabled AND its in-window crash restarts are below the cap.
885///
886/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
887/// the window are dropped here rather than by a timer, so the count is right
888/// the moment somebody asks and no bookkeeping runs for idle modules.
889fn daemon_will_restart(
890    state: &mut SupervisorSnapshot,
891    policy: &RestartPolicy,
892    now: Instant,
893) -> bool {
894    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
895}
896
897const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
898const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
899const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
900const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
901
902#[derive(Debug, Clone, Copy, PartialEq, Eq)]
903pub enum HealthAction {
904    Report,
905    Restart,
906    Alert,
907}
908
909impl fmt::Display for HealthAction {
910    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
911        f.write_str(match self {
912            Self::Report => "report",
913            Self::Restart => "restart",
914            Self::Alert => "alert",
915        })
916    }
917}
918
919#[derive(Debug, Clone, PartialEq, Eq)]
920pub struct HealthConfig {
921    /// Optional loopback HTTP endpoint for a managed non-wire process.
922    /// Changing it applies live on rescan; the process protocol changes only
923    /// at its next spawn.
924    pub http: Option<String>,
925    pub cadence: Duration,
926    pub deadline: Duration,
927    pub failure_threshold: u32,
928    pub on_degraded: HealthAction,
929    pub on_failing: HealthAction,
930    pub critical: bool,
931}
932
933impl Default for HealthConfig {
934    fn default() -> Self {
935        Self {
936            http: None,
937            cadence: DEFAULT_HEALTH_CADENCE,
938            deadline: DEFAULT_HEALTH_DEADLINE,
939            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
940            on_degraded: HealthAction::Report,
941            on_failing: HealthAction::Report,
942            critical: false,
943        }
944    }
945}
946
947/// The supervisor's view of one module's health, relayed to clients over
948/// channel-0 and rendered by `ck health`.
949///
950/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
951/// stated here rather than only at the wire type a consumer reads. A reader can
952/// look up what `None` means; only a writer can silently change it, and the
953/// writer has no reason to go looking at a downstream contract before editing.
954///
955/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
956/// back to `None` on re-registration precisely so a respawned module does not
957/// carry its predecessor's timestamp — so an old value and an absent one call for
958/// opposite readings, and anything that defaulted this to a number would make a
959/// never-probed module indistinguishable from one probed at the epoch.
960///
961/// `detail` and `metrics` are `None` when the module published none on this
962/// probe, which does not mean it reported nothing wrong — it is also the shape
963/// when the probe never reached it. `last_probe_ms` is what separates those.
964#[derive(Debug, Clone, PartialEq)]
965pub struct ModuleHealthStatus {
966    pub status: SupervisorHealthStatus,
967    pub last_probe_ms: Option<u64>,
968    pub detail: Option<String>,
969    pub metrics: Option<Value>,
970    pub consecutive_failures: u32,
971    /// Number of replies received after a recurring health probe's deadline.
972    /// Unlike a timeout, every increment proves the module was alive.
973    pub late_answer_count: u64,
974    /// End-to-end latency of the newest late reply, measured from probe start.
975    pub last_late_answer_latency_ms: Option<u64>,
976    pub last_action: Option<String>,
977    /// Set together with `last_action`; the pair moves as one, and both being
978    /// absent means no escalation has ever been taken rather than that the last
979    /// one succeeded.
980    pub last_action_ms: Option<u64>,
981}
982
983impl Default for ModuleHealthStatus {
984    fn default() -> Self {
985        Self {
986            status: SupervisorHealthStatus::Unknown,
987            last_probe_ms: None,
988            detail: None,
989            metrics: None,
990            consecutive_failures: 0,
991            late_answer_count: 0,
992            last_late_answer_latency_ms: None,
993            last_action: None,
994            last_action_ms: None,
995        }
996    }
997}
998
999/// Typed lifecycle state for a supervised module.
1000#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1001pub enum ModuleState {
1002    Starting,
1003    Running,
1004    Unresponsive,
1005    Restarting,
1006    Draining,
1007    Stopped,
1008    Failed,
1009    Disabled,
1010}
1011
1012impl fmt::Display for ModuleState {
1013    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1014        f.write_str(match self {
1015            Self::Starting => "starting",
1016            Self::Running => "running",
1017            Self::Unresponsive => "unresponsive",
1018            Self::Restarting => "restarting",
1019            Self::Draining => "draining",
1020            Self::Stopped => "stopped",
1021            Self::Failed => "failed",
1022            Self::Disabled => "disabled",
1023        })
1024    }
1025}
1026
1027/// Supervisor classification of a child-process exit.
1028#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1029pub enum ExitKind {
1030    Clean,
1031    Crash,
1032    DeliberateSeverance,
1033}
1034
1035impl From<ExitKind> for TerminalExitKind {
1036    fn from(kind: ExitKind) -> Self {
1037        match kind {
1038            ExitKind::Clean => Self::Clean,
1039            ExitKind::Crash => Self::Crash,
1040            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1041        }
1042    }
1043}
1044
1045/// Exact process identity retained when a supervised module registers its
1046/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1047#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1048pub(crate) struct ProcessIdentity {
1049    pub(crate) pid: u32,
1050    pub(crate) start_time: u64,
1051}
1052
1053/// Last observed child exit, if any.
1054#[derive(Debug, Clone, PartialEq, Eq)]
1055pub struct ExitReport {
1056    pub kind: ExitKind,
1057    pub code: Option<i32>,
1058    pub signal: Option<i32>,
1059    pub at_ms: u64,
1060}
1061
1062/// Point-in-time module status answerable by subc without forwarding to the
1063/// module process.
1064#[derive(Debug, Clone, PartialEq)]
1065pub struct ModuleStatus {
1066    pub module_id: String,
1067    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1068    pub waiting_for_previous_pid: Option<u32>,
1069    /// Why the previous instance still prevents a new start.
1070    pub waiting_for_previous_detail: Option<String>,
1071    /// The daemon's Windows per-spawn record: `pipe` after the process it
1072    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1073    /// List/status reports use this record, not the source the module sends
1074    /// in its registration message's manifest.provenance.launch_nonce_source.
1075    pub launch_nonce_source: Option<String>,
1076    pub state: ModuleState,
1077    pub enabled: bool,
1078    pub process_alive: bool,
1079    pub registration_active: bool,
1080    /// The module's declared wire protocol, carried beside `live` because it is
1081    /// what makes `live` readable: the two fields answer one question together.
1082    /// While a process is alive this is its launch declaration, not a later
1083    /// pending-reload edit. When down it is the configured next launch protocol.
1084    pub protocol: ModuleProtocol,
1085    /// Whether the module is serving, under the strongest definition the daemon
1086    /// can assert for its protocol.
1087    ///
1088    /// A subc module must also be REGISTERED: its process being alive says
1089    /// nothing about whether it can take a request. A `protocol: "none"` module
1090    /// never registers, so that term is dropped and this falls back to "enabled,
1091    /// running, and the process the daemon launched is alive" -- which is all
1092    /// the daemon observes about a process that speaks no subc wire. It stays a
1093    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1094    /// rather than printing it bare.
1095    pub live: bool,
1096    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1097    /// restarts have already released their slot, so this count can go down
1098    /// without anybody touching the module.
1099    pub restart_count: u32,
1100    /// Replacement processes spawned over this module's entire supervisor lifetime;
1101    /// unlike `restart_count`, this value is never reset by an operator action
1102    /// and never falls out of a window.
1103    pub lifetime_restarts: u32,
1104    pub spawn_generation: u64,
1105    /// The budget `restart_count` is spent against. Carried alongside the count
1106    /// because the count alone does not say how close the module is to being
1107    /// disabled, and reporting one without the other is what makes an
1108    /// about-to-be-retired module look ordinary.
1109    pub max_restarts: u32,
1110    /// The span `restart_count` is counted over. Carried with the pair above for
1111    /// the same reason they are carried together: "2 of 3" means one thing for a
1112    /// ten-minute window and something else entirely for a lifetime.
1113    pub restart_window: Duration,
1114    /// Effective drain and restart timing policy used by this running module.
1115    /// These values are carried together with the restart budget so status
1116    /// readers can compare configured intent with what the supervisor applied.
1117    pub drain_timeout: Duration,
1118    pub restart_backoff: Duration,
1119    pub restart_max_backoff: Duration,
1120    /// The module's process. On macOS this stays absent while the `ck-subc`
1121    /// launch trampoline is still running in that pid, and appears once the
1122    /// exec acknowledgement confirms the module image has replaced it. Launch
1123    /// time and the supervisor's own hold on the process are unaffected.
1124    pub pid: Option<u32>,
1125    pub spawned_at_ms: Option<u64>,
1126    pub spawned_from: Option<PathBuf>,
1127    pub process_start_time: Option<u64>,
1128    pub last_exit: Option<ExitReport>,
1129    pub health: ModuleHealthStatus,
1130}
1131
1132#[cfg(test)]
1133#[derive(Debug, Clone, Copy, PartialEq)]
1134struct ActorSelectCheckpoint {
1135    generation: u64,
1136    turn: u64,
1137    registered_connection: Option<ConnectionId>,
1138    next_probe_at: Option<Instant>,
1139    wake_after: Option<Duration>,
1140}
1141
1142#[derive(Debug, Clone, PartialEq)]
1143struct SupervisorSnapshot {
1144    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1145    #[cfg(windows)]
1146    pipe_consumed: Option<Arc<OnceLock<()>>>,
1147    /// Counts running-child loop turns, not executor polls of a parked wait.
1148    #[cfg(test)]
1149    actor_turns: u64,
1150    /// Running select after exec confirmation and with no unconsumed registry
1151    /// event. Recording a spawn or starting a loop turn is not this barrier.
1152    #[cfg(test)]
1153    actor_select: Option<ActorSelectCheckpoint>,
1154    state: ModuleState,
1155    enabled: bool,
1156    process_alive: bool,
1157    spawned_protocol: Option<ModuleProtocol>,
1158    spawn_failure: Option<String>,
1159    waiting_for_previous_pid: Option<u32>,
1160    waiting_for_previous_detail: Option<String>,
1161    /// When each crash restart was spent, oldest first. This IS the crash
1162    /// budget: its in-window length is the count an operator sees and the count
1163    /// the restart decision is made against, so there is no second counter that
1164    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1165    /// operator actions that used to zero the old lifetime counter.
1166    crash_restarts: VecDeque<Instant>,
1167    lifetime_restarts: u32,
1168    /// Successful child spawns in this daemon incarnation.
1169    ///
1170    /// `lifetime_restarts` was considered and rejected: it starts at zero
1171    /// (line 640), successful initial/operator spawns in `set_running` do not
1172    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1173    /// increments before a successful replacement exists (lines 604, 3846,
1174    /// and 3921), so a failed spawn can consume it. This counter moves only
1175    /// when a live PID is accepted below.
1176    spawn_generation: u64,
1177    pid: Option<u32>,
1178    #[cfg(target_os = "macos")]
1179    report_ready: Option<Arc<OnceLock<()>>>,
1180    /// Last reaped child, retained after current process facts are cleared.
1181    reaped_pid: Option<u32>,
1182    /// Whether a replacement has already been scheduled for the current process.
1183    respawn_pending: bool,
1184    /// A second restart is waiting for the replacement already scheduled.
1185    coalesced_restart_pending: bool,
1186    spawned_at_ms: Option<u64>,
1187    spawned_from: Option<PathBuf>,
1188    spawned_file_identity: Option<SpawnedFileIdentity>,
1189    process_start_time: Option<u64>,
1190    deliberate_severance: Option<ProcessIdentity>,
1191    last_exit: Option<ExitReport>,
1192    /// Diagnostic attached to the next drain's terminal record, if any.
1193    drain_disposition_detail: Option<String>,
1194    health: ModuleHealthStatus,
1195    /// Whether the current process was started as a swap candidate and so
1196    /// lives in the module's alternate cgroup. The next swap's candidate takes
1197    /// the other one, so the two processes of a swap never share a cgroup. A
1198    /// plain spawn always uses the primary cgroup.
1199    in_alternate_slot: bool,
1200    /// Whether the current `Draining` state ends in a replacement process
1201    /// (restart, reload, health restart) rather than a stop. Only meaningful
1202    /// while `state` is `Draining`; every entry into that state rewrites it.
1203    /// It is what lets route.open answer the retryable `module_reloading` to a
1204    /// consumer that reaches a still-registered process mid-restart, instead of
1205    /// the `supervisor_not_live` a stop or disable deserves.
1206    draining_to_replace: bool,
1207    /// Whether a configuration update has been applied since the current
1208    /// process was spawned, so that process runs an older spec than the one
1209    /// the supervisor now holds. A queued restart is only coalesced into a
1210    /// fresher process when this is false: a restart requested to pick up a
1211    /// new configuration must not be satisfied by a process that predates it.
1212    configuration_updated_since_spawn: bool,
1213}
1214
1215impl SupervisorSnapshot {
1216    /// The pid that status, provenance and resource readings may report. While
1217    /// the launch trampoline still runs in the pid, reading its executable or
1218    /// resource use would describe `ck-subc`, not the module, so none is
1219    /// reported until the exec acknowledgement confirms the module image.
1220    fn reported_pid(&self) -> Option<u32> {
1221        #[cfg(target_os = "macos")]
1222        if self
1223            .report_ready
1224            .as_ref()
1225            .is_some_and(|ready| ready.get().is_none())
1226        {
1227            return None;
1228        }
1229        self.pid
1230    }
1231
1232    fn starting() -> Self {
1233        Self::new(ModuleState::Starting, true)
1234    }
1235
1236    fn disabled() -> Self {
1237        Self::new(ModuleState::Disabled, false)
1238    }
1239
1240    /// Crash restarts still inside `window`, having dropped the ones that are
1241    /// not. Pruning on read is what makes the budget a rate: an instant older
1242    /// than the window stops holding a slot the moment anybody counts.
1243    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1244        while let Some(oldest) = self.crash_restarts.front() {
1245            if now.duration_since(*oldest) > window {
1246                self.crash_restarts.pop_front();
1247            } else {
1248                break;
1249            }
1250        }
1251        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1252    }
1253
1254    /// Spend one unit of the crash budget and record the restart in the ledger.
1255    ///
1256    /// The ring is bounded by the cap because more than `max_restarts` in-window
1257    /// instants can never be reached (the caller refuses the restart first), so
1258    /// anything beyond that is an unbounded queue waiting to happen.
1259    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1260        self.crash_restarts.push_back(now);
1261        while self.crash_restarts.len() > policy.max_restarts as usize {
1262            self.crash_restarts.pop_front();
1263        }
1264        self.lifetime_restarts += 1;
1265    }
1266
1267    /// Reserve one crash-restart slot and calculate the delay before respawning.
1268    /// The count is captured before recording this restart, so the first retry
1269    /// uses the base delay and each later in-window retry escalates once.
1270    fn next_crash_restart(
1271        &mut self,
1272        policy: &RestartPolicy,
1273        now: Instant,
1274    ) -> Option<CrashRestartSchedule> {
1275        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1276        if restart_in_window >= policy.max_restarts {
1277            return None;
1278        }
1279        self.record_crash_restart(policy, now);
1280        Some(CrashRestartSchedule {
1281            restart_in_window,
1282            delay: policy.delay_for_restart(restart_in_window),
1283        })
1284    }
1285
1286    /// Give the module its full budget back, as an operator restart, reload, or
1287    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1288    /// ledger of what actually happened, and an operator action does not unmake
1289    /// the crashes.
1290    fn clear_crash_restarts(&mut self) {
1291        self.crash_restarts.clear();
1292    }
1293
1294    fn new(state: ModuleState, enabled: bool) -> Self {
1295        Self {
1296            exec_outcome: None,
1297            #[cfg(test)]
1298            actor_turns: 0,
1299            #[cfg(test)]
1300            actor_select: None,
1301            state,
1302            enabled,
1303            process_alive: false,
1304            #[cfg(windows)]
1305            pipe_consumed: None,
1306            spawned_protocol: None,
1307            spawn_failure: None,
1308            waiting_for_previous_pid: None,
1309            waiting_for_previous_detail: None,
1310            crash_restarts: VecDeque::new(),
1311            lifetime_restarts: 0,
1312            spawn_generation: 0,
1313            pid: None,
1314            #[cfg(target_os = "macos")]
1315            report_ready: None,
1316            reaped_pid: None,
1317            respawn_pending: false,
1318            coalesced_restart_pending: false,
1319            spawned_at_ms: None,
1320            spawned_from: None,
1321            spawned_file_identity: None,
1322            process_start_time: None,
1323            deliberate_severance: None,
1324            last_exit: None,
1325            drain_disposition_detail: None,
1326            health: ModuleHealthStatus::default(),
1327            in_alternate_slot: false,
1328            draining_to_replace: false,
1329            configuration_updated_since_spawn: false,
1330        }
1331    }
1332}
1333
1334type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1335
1336type SpawnSubscriberKey = (ConnectionId, u64);
1337
1338#[derive(Debug)]
1339struct SpawnSubscriber {
1340    version: u8,
1341    frames: mpsc::Sender<Frame>,
1342    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1343    /// from which event. The full frame channel cannot carry that news, so it
1344    /// travels beside it; see `SpawnEventFeed::subscribe`.
1345    lagged: Option<oneshot::Sender<SpawnCursor>>,
1346}
1347
1348#[derive(Debug)]
1349struct SpawnEventState {
1350    daemon_incarnation: String,
1351    seq: u64,
1352    capacity: usize,
1353    live: HashMap<String, LiveSpawn>,
1354    generations: HashMap<String, u64>,
1355    events: VecDeque<SpawnEvent>,
1356    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1357}
1358
1359impl Default for SpawnEventState {
1360    fn default() -> Self {
1361        Self {
1362            daemon_incarnation: "unconfigured".to_string(),
1363            seq: 0,
1364            capacity: SPAWN_EVENT_RING_CAPACITY,
1365            live: HashMap::new(),
1366            generations: HashMap::new(),
1367            events: VecDeque::new(),
1368            subscribers: HashMap::new(),
1369        }
1370    }
1371}
1372
1373#[derive(Debug, Clone, Default)]
1374struct SpawnEventFeed(
1375    Arc<Mutex<SpawnEventState>>,
1376    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1377);
1378
1379#[derive(Debug, Clone, PartialEq, Eq)]
1380pub(crate) enum SpawnSubscribeRefusal {
1381    ForeignIncarnation { current: String },
1382    TooOld { oldest: SpawnCursor },
1383    Frame(String),
1384}
1385
1386impl SpawnEventFeed {
1387    fn configure_incarnation(&self, daemon_incarnation: String) {
1388        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1389        state.daemon_incarnation = daemon_incarnation;
1390        state.seq = 0;
1391        state.live.clear();
1392        state.generations.clear();
1393        state.events.clear();
1394        state.subscribers.clear();
1395    }
1396
1397    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1398        SpawnCursor {
1399            daemon_incarnation: state.daemon_incarnation.clone(),
1400            seq: state.seq,
1401        }
1402    }
1403
1404    fn snapshot(&self) -> SpawnSnapshot {
1405        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1406        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1407        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1408        SpawnSnapshot {
1409            cursor: Self::cursor(&state),
1410            ring_bound: state.capacity as u64,
1411            live,
1412        }
1413    }
1414
1415    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1416        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1417        let generation = state
1418            .generations
1419            .get(module_id)
1420            .copied()
1421            .unwrap_or(0)
1422            .checked_add(1)
1423            .expect("spawn generation exhausted");
1424        state.generations.insert(module_id.to_string(), generation);
1425        let live = LiveSpawn {
1426            module_id: module_id.to_string(),
1427            spawn_generation: generation,
1428            pid,
1429            spawned_at_ms,
1430        };
1431        state.live.insert(module_id.to_string(), live);
1432        Self::emit_locked(
1433            &mut state,
1434            SpawnEventKind::Spawned,
1435            module_id.to_string(),
1436            generation,
1437            pid,
1438            None,
1439            None,
1440        );
1441        generation
1442    }
1443
1444    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1445        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1446        let Some(live) = state.live.remove(module_id) else {
1447            warn!(
1448                module_id,
1449                "terminal record had no live spawn event identity"
1450            );
1451            return;
1452        };
1453        Self::emit_locked(
1454            &mut state,
1455            SpawnEventKind::Exited,
1456            module_id.to_string(),
1457            live.spawn_generation,
1458            live.pid,
1459            exit_code,
1460            exit_signal,
1461        );
1462    }
1463
1464    /// Report the exit of a process that a swap has already replaced.
1465    ///
1466    /// `emit_exited` removes the module's live entry, which after a swap's
1467    /// cutover describes the promoted candidate, not the old process now
1468    /// exiting. This emits the old generation's exit and leaves the live entry
1469    /// alone unless it still names that generation.
1470    fn emit_superseded_exited(
1471        &self,
1472        module_id: &str,
1473        spawn_generation: u64,
1474        pid: u32,
1475        exit_code: Option<i32>,
1476        exit_signal: Option<i32>,
1477    ) {
1478        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1479        if state
1480            .live
1481            .get(module_id)
1482            .is_some_and(|live| live.spawn_generation == spawn_generation)
1483        {
1484            state.live.remove(module_id);
1485        }
1486        Self::emit_locked(
1487            &mut state,
1488            SpawnEventKind::Exited,
1489            module_id.to_string(),
1490            spawn_generation,
1491            pid,
1492            exit_code,
1493            exit_signal,
1494        );
1495    }
1496
1497    #[allow(clippy::too_many_arguments)]
1498    fn emit_locked(
1499        state: &mut SpawnEventState,
1500        kind: SpawnEventKind,
1501        module_id: String,
1502        spawn_generation: u64,
1503        pid: u32,
1504        exit_code: Option<i32>,
1505        exit_signal: Option<i32>,
1506    ) {
1507        state.seq = state
1508            .seq
1509            .checked_add(1)
1510            .expect("spawn event sequence exhausted");
1511        let event = SpawnEvent {
1512            cursor: Self::cursor(state),
1513            kind,
1514            module_id,
1515            spawn_generation,
1516            pid,
1517            exit_code,
1518            exit_signal,
1519        };
1520        state.events.push_back(event.clone());
1521        while state.events.len() > state.capacity {
1522            state.events.pop_front();
1523        }
1524        let body = match serde_json::to_vec(&event) {
1525            Ok(body) => body,
1526            Err(error) => {
1527                error!(%error, "failed to serialize supervisor spawn event");
1528                return;
1529            }
1530        };
1531        state.subscribers.retain(|(connection_id, corr), subscriber| {
1532            let frame = Frame::build_with_version(
1533                subscriber.version,
1534                FrameType::StreamData,
1535                control_flags(),
1536                0,
1537                0,
1538                *corr,
1539                body.clone(),
1540            );
1541            match frame {
1542                Ok(frame) => {
1543                    if subscriber.frames.try_send(frame).is_ok() {
1544                        true
1545                    } else {
1546                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1547                        if let Some(lagged) = subscriber.lagged.take() {
1548                            let _ = lagged.send(event.cursor.clone());
1549                        }
1550                        false
1551                    }
1552                }
1553                Err(error) => {
1554                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1555                    false
1556                }
1557            }
1558        });
1559    }
1560
1561    fn subscribe(
1562        &self,
1563        connection_id: ConnectionId,
1564        corr: u64,
1565        version: u8,
1566        since: Option<SpawnCursor>,
1567        sink: FrameSink,
1568    ) -> Result<(), SpawnSubscribeRefusal> {
1569        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1570        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1571        {
1572            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1573            let replay = if let Some(since) = since {
1574                if since.daemon_incarnation != state.daemon_incarnation {
1575                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1576                        current: state.daemon_incarnation.clone(),
1577                    });
1578                }
1579                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1580                    if since.seq < oldest.seq.saturating_sub(1) {
1581                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1582                    }
1583                }
1584                state
1585                    .events
1586                    .iter()
1587                    .filter(|event| event.cursor.seq > since.seq)
1588                    .cloned()
1589                    .collect::<Vec<_>>()
1590            } else {
1591                Vec::new()
1592            };
1593            for event in replay {
1594                let body = serde_json::to_vec(&event)
1595                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1596                let frame = Frame::build_with_version(
1597                    version,
1598                    FrameType::StreamData,
1599                    control_flags(),
1600                    0,
1601                    0,
1602                    corr,
1603                    body,
1604                )
1605                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1606                frames
1607                    .try_send(frame)
1608                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1609            }
1610            state.subscribers.insert(
1611                (connection_id, corr),
1612                SpawnSubscriber {
1613                    version,
1614                    frames,
1615                    lagged: Some(lagged),
1616                },
1617            );
1618        }
1619        // The lagged terminal is sent here, by the forwarder, rather than by
1620        // the emitter: at the moment of the drop the subscriber's own channel
1621        // is full, and writing to the connection sink directly from the emitter
1622        // would put the Error AHEAD of the events still queued in that channel
1623        // (and the emitter holds the feed lock, so it cannot await the sink).
1624        // Dropping the subscriber drops the only sender, so `recv` drains every
1625        // queued event and then returns `None`; only then is the Error sent, so
1626        // the client sees each event it can keep, then the reason it was cut.
1627        // Cancel and connection removal drop the oneshot unsent, so they end
1628        // the stream with no Error.
1629        tokio::spawn(async move {
1630            while let Some(frame) = receiver.recv().await {
1631                if sink.send(frame).await.is_err() {
1632                    return;
1633                }
1634            }
1635            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1636                return;
1637            };
1638            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1639                Ok(frame) => {
1640                    let _ = sink.send(frame).await;
1641                }
1642                Err(error) => {
1643                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1644                }
1645            }
1646        });
1647        Ok(())
1648    }
1649
1650    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1651        let Some(subscriber) = self
1652            .0
1653            .lock()
1654            .unwrap_or_else(|p| p.into_inner())
1655            .subscribers
1656            .remove(&(connection_id, corr))
1657        else {
1658            return false;
1659        };
1660        if let Ok(frame) = Frame::build_with_version(
1661            subscriber.version,
1662            FrameType::StreamEnd,
1663            control_flags(),
1664            0,
1665            0,
1666            corr,
1667            Vec::new(),
1668        ) {
1669            tokio::spawn(async move {
1670                let _ = subscriber.frames.send(frame).await;
1671            });
1672        }
1673        true
1674    }
1675
1676    fn remove_connection(&self, connection_id: ConnectionId) {
1677        self.0
1678            .lock()
1679            .unwrap_or_else(|p| p.into_inner())
1680            .subscribers
1681            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1682    }
1683
1684    #[cfg(any(test, feature = "test-support"))]
1685    fn set_capacity(&self, capacity: usize) {
1686        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1687    }
1688
1689    #[cfg(any(test, feature = "test-support"))]
1690    fn subscriber_count(&self) -> usize {
1691        self.0
1692            .lock()
1693            .unwrap_or_else(|p| p.into_inner())
1694            .subscribers
1695            .len()
1696    }
1697}
1698
1699/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1700/// The terminal Error a lagged spawn subscriber receives after its queued events.
1701fn spawn_subscriber_lagged_frame(
1702    version: u8,
1703    corr: u64,
1704    first_undelivered: SpawnCursor,
1705) -> Result<Frame, String> {
1706    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1707        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1708        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1709            .to_string(),
1710        detail: Some(serde_json::json!({
1711            "first_undelivered_cursor": first_undelivered
1712        })),
1713    })
1714    .map_err(|error| error.to_string())?;
1715    Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1716        .map_err(|error| error.to_string())
1717}
1718
1719pub trait ModuleProcessLiveness: Send + Sync {
1720    fn process_live(&self, module_id: &str) -> Option<bool>;
1721
1722    /// Whether the supervisor is replacing this module's process right now: an
1723    /// operator restart or reload, a health restart, or a crash respawn whose
1724    /// backoff is running. A module in that state is not live, but a consumer
1725    /// refused now should retry shortly rather than treat the target as gone.
1726    /// Stopped, failed, and disabled modules are not replacing.
1727    fn process_replacing(&self, _module_id: &str) -> bool {
1728        false
1729    }
1730}
1731
1732/// Shared process-liveness registry keyed by supervised `module_id`.
1733#[derive(Debug, Clone, Default)]
1734pub struct SupervisorProcessLiveness {
1735    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1736}
1737
1738impl SupervisorProcessLiveness {
1739    pub fn new() -> Self {
1740        Self::default()
1741    }
1742
1743    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1744        let mut snapshots = self
1745            .snapshots
1746            .lock()
1747            .unwrap_or_else(|poisoned| poisoned.into_inner());
1748        snapshots.insert(module_id, snapshot);
1749    }
1750
1751    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1752        let mut snapshots = self
1753            .snapshots
1754            .lock()
1755            .unwrap_or_else(|poisoned| poisoned.into_inner());
1756        let is_current = snapshots
1757            .get(module_id)
1758            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1759            .unwrap_or(false);
1760        if is_current {
1761            snapshots.remove(module_id);
1762        }
1763    }
1764}
1765
1766impl ModuleProcessLiveness for SupervisorProcessLiveness {
1767    fn process_live(&self, module_id: &str) -> Option<bool> {
1768        let snapshot = {
1769            let snapshots = self
1770                .snapshots
1771                .lock()
1772                .unwrap_or_else(|poisoned| poisoned.into_inner());
1773            snapshots.get(module_id).cloned()
1774        }?;
1775        let snapshot = snapshot
1776            .lock()
1777            .unwrap_or_else(|poisoned| poisoned.into_inner());
1778        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1779    }
1780
1781    fn process_replacing(&self, module_id: &str) -> bool {
1782        let Some(snapshot) = self
1783            .snapshots
1784            .lock()
1785            .unwrap_or_else(|poisoned| poisoned.into_inner())
1786            .get(module_id)
1787            .cloned()
1788        else {
1789            return false;
1790        };
1791        let snapshot = snapshot
1792            .lock()
1793            .unwrap_or_else(|poisoned| poisoned.into_inner());
1794        snapshot.enabled
1795            && match snapshot.state {
1796                ModuleState::Restarting => true,
1797                ModuleState::Draining => snapshot.draining_to_replace,
1798                ModuleState::Starting
1799                | ModuleState::Running
1800                | ModuleState::Unresponsive
1801                | ModuleState::Stopped
1802                | ModuleState::Failed
1803                | ModuleState::Disabled => false,
1804            }
1805    }
1806}
1807
1808#[cfg(test)]
1809#[derive(Debug, Default)]
1810struct ReloadExitRecordGate {
1811    reached: tokio::sync::Notify,
1812    resume: tokio::sync::Notify,
1813}
1814
1815#[derive(Debug, Clone, Copy)]
1816enum RespawnKind {
1817    Spawn,
1818    Reload,
1819}
1820
1821#[derive(Debug, Clone, Copy)]
1822struct PendingRespawn {
1823    deadline: Instant,
1824    kind: RespawnKind,
1825}
1826
1827type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1828
1829#[derive(Debug, Clone)]
1830struct SupervisorRuntimeConfig {
1831    start_requester: Option<Principal>,
1832    pending_activity_restart: bool,
1833    provenance_probe: ExecutableIdentityProbe,
1834    /// Restart operations hand their backoff to the loop, which keeps serving commands.
1835    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1836    /// A reload acknowledges completion only after its replacement registers.
1837    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1838    restart_policy: RestartPolicy,
1839    /// This module's RESOLVED drain budget: per-module config when present,
1840    /// else `default_drain_timeout`.
1841    drain_timeout: Duration,
1842    /// Shared with the status handle so the attested value changes atomically
1843    /// when a rescan updates the running drain policy.
1844    effective_drain_timeout: Arc<Mutex<Duration>>,
1845    /// The supervisor-wide fallback, kept so a configuration update that
1846    /// REMOVES the per-module override can re-resolve to it.
1847    default_drain_timeout: Duration,
1848    health: HealthConfig,
1849    connection_file_path: Option<PathBuf>,
1850    capture_logs_dir: Option<PathBuf>,
1851    forwarding: Option<Arc<ForwardingTable>>,
1852    /// The shared handle, so every spawn path (initial, restart, reload) records the
1853    /// reserved-module launch nonce the HELLO verifier checks against.
1854    supervisor_handle: Option<SupervisorHandle>,
1855    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
1856    /// status queries.
1857    ///
1858    /// One ring per module, held across every respawn. The lines explaining an exit
1859    /// are written BEFORE that exit, so a ring recreated per process would be empty
1860    /// exactly when it is asked for.
1861    stderr_ring: Arc<Mutex<StderrRing>>,
1862    terminal_ring: Arc<Mutex<TerminalRing>>,
1863    spawn_events: SpawnEventFeed,
1864    child_roster: ChildRoster,
1865    #[cfg(target_os = "linux")]
1866    cgroup_placement: Option<subc_cgroup::Placement>,
1867    #[cfg(test)]
1868    test_seed_stale_facts_before_enable_spawn: bool,
1869    #[cfg(test)]
1870    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1871}
1872
1873#[derive(Debug, Clone, PartialEq, Eq)]
1874struct SupervisedConfiguration {
1875    spec: ModuleSpec,
1876    health: HealthConfig,
1877}
1878
1879/// Shared daemon lookup table for supervised module handles.
1880///
1881/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
1882/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
1883/// launch nonces recorded at spawn are checked by the same daemon instance.
1884#[derive(Debug, Clone, Default)]
1885pub struct SupervisorHandle {
1886    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1887    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1888    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1889    /// Module ids the supervisor has taken on. An id is added BEFORE the
1890    /// module's first process is spawned and removed only when the module
1891    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
1892    /// the keys of `modules`.
1893    ///
1894    /// `modules` cannot answer "is this module configured?" on its own: a
1895    /// [`SupervisedModule`] only exists once its process has been spawned, and
1896    /// a fast child can connect, register, sync its scopes and ask about them
1897    /// before the supervisor has inserted it. Answering "not configured" in that
1898    /// gap makes scope admission refuse with the terminal "will never sync"
1899    /// instead of the retryable "has not synced yet".
1900    configured_ids: Arc<Mutex<HashSet<String>>>,
1901    spawn_events: SpawnEventFeed,
1902    /// The current expected launch nonce for each reserved module_id. Set when the
1903    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
1904    /// non-reserved module never has an entry here and is never nonce-checked.
1905    /// Reserved module ids and the nonce that authorizes their next HELLO.
1906    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
1907    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
1908    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
1909    /// had NO entry and admitted anyone: the reservation protected the nonce
1910    /// holder, not the NAME (found live by CKCRED's canary probe registering
1911    /// against a reserved scratch id).
1912    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
1913    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
1914    ///
1915    /// This is deliberately in-memory only: subc is state-free across daemon
1916    /// restarts, and the tombstone only explains the hours-after-removal window
1917    /// while this executing daemon is still alive. Do not persist it in a store.
1918    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
1919    /// The current launch nonce for every supervised spawn. This is separate from
1920    /// reserved_nonces because consumer route.open attestation applies to all spawned
1921    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
1922    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
1923    /// Reserved namespace prefixes mapped to the supervised owner module whose
1924    /// current spawn nonce authorizes HELLO claims below the prefix.
1925    ///
1926    /// Per §2.6 this is not a same-user security barrier: a same-user process can
1927    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
1928    /// accidental collisions and lower-trust processes from squatting protected
1929    /// namespaces.
1930    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
1931    /// Blue/green swaps in progress, by module id. An entry exists from just
1932    /// before the candidate process is spawned until the swap has failed, or
1933    /// has cut over and the old process is gone. While it exists, HELLO for the
1934    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
1935    /// consumer attestation accepts both processes' nonces.
1936    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
1937    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
1938    promotion_observer: PromotionObserverSlot,
1939    /// Serializes module-set reconciliation with operator lifecycle commands. Without
1940    /// this daemon-wide ordering, a rescan could retire or update a module while a
1941    /// concurrent reload still held its old handle and launch specification.
1942    operation_lock: Arc<AsyncMutex<()>>,
1943}
1944
1945/// Told when a swap has promoted its candidate to be the module's active
1946/// registration.
1947///
1948/// An ordinary HELLO runs the control plane's registration side effects (the
1949/// capability cache, the deny census, the requirement recompute) as it
1950/// registers. A swap candidate's HELLO does not, because it is not routable;
1951/// promotion is when those must run instead, and promotion happens in the
1952/// supervisor, which has no other way into the control handler.
1953pub(crate) trait SwapPromotionObserver: Send + Sync {
1954    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
1955}
1956
1957/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
1958/// control handler) owns this handle, so a strong reference back would be a
1959/// cycle that keeps both alive.
1960#[derive(Clone, Default)]
1961struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
1962
1963impl fmt::Debug for PromotionObserverSlot {
1964    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1965        f.write_str("PromotionObserverSlot")
1966    }
1967}
1968
1969/// The nonces of one open swap.
1970#[derive(Debug, Clone)]
1971struct OpenSwap {
1972    /// The launch nonce minted for the candidate process. It is the swap
1973    /// token: the only thing that admits a HELLO into the candidate slot.
1974    candidate_nonce: String,
1975    /// The incumbent's launch nonce, captured when the swap opened. It is kept
1976    /// here because cutover moves the module's recorded spawn nonce to the
1977    /// candidate while the incumbent is still draining and its consumers are
1978    /// still attesting with this one.
1979    incumbent_nonce: Option<String>,
1980    /// Set once a HELLO has been admitted with the swap token, so the token
1981    /// admits one registration and cannot be replayed after cutover empties
1982    /// the candidate slot.
1983    candidate_admitted: bool,
1984}
1985
1986/// What the swap gate says about a HELLO. See
1987/// [`SupervisorHandle::swap_hello_admission`].
1988#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1989pub(crate) enum SwapHelloAdmission {
1990    /// No swap is open for the id (or the HELLO carries the incumbent's own
1991    /// nonce); the ordinary gates decide.
1992    NotSwapping,
1993    /// The HELLO carries the swap token: register it into the candidate slot.
1994    Candidate,
1995    /// A swap is open and the HELLO carries a nonce the supervisor did not
1996    /// mint for this id, no nonce, or a token already used.
1997    Refused,
1998}
1999
2000#[derive(Debug, Clone, PartialEq, Eq)]
2001pub(crate) enum ReservedHelloRejection {
2002    Exact {
2003        module_id: String,
2004    },
2005    Prefix {
2006        prefix: String,
2007        owner_module_id: String,
2008    },
2009}
2010
2011impl SupervisorHandle {
2012    pub fn new() -> Self {
2013        Self::default()
2014    }
2015
2016    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2017        self.spawn_events.snapshot()
2018    }
2019
2020    pub(crate) fn subscribe_spawns(
2021        &self,
2022        connection_id: ConnectionId,
2023        corr: u64,
2024        version: u8,
2025        since: Option<SpawnCursor>,
2026        sink: FrameSink,
2027    ) -> Result<(), SpawnSubscribeRefusal> {
2028        self.spawn_events
2029            .subscribe(connection_id, corr, version, since, sink)
2030    }
2031
2032    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2033        self.spawn_events.cancel(connection_id, corr)
2034    }
2035
2036    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2037        self.spawn_events.remove_connection(connection_id);
2038    }
2039
2040    #[cfg(any(test, feature = "test-support"))]
2041    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2042        assert!(capacity > 0, "spawn event capacity must be non-zero");
2043        self.spawn_events.set_capacity(capacity);
2044    }
2045
2046    #[cfg(any(test, feature = "test-support"))]
2047    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2048        self.spawn_events.subscriber_count()
2049    }
2050
2051    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2052    /// a respawn invalidates stale consumer identities.
2053    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2054        self.spawn_nonces
2055            .lock()
2056            .unwrap_or_else(|poisoned| poisoned.into_inner())
2057            .insert(module_id.to_string(), nonce);
2058    }
2059
2060    /// Record the launch nonce expected from the next HELLO for a reserved module,
2061    /// replacing any prior nonce (a respawn invalidates the previous one).
2062    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2063        self.reserved_nonces
2064            .lock()
2065            .unwrap_or_else(|poisoned| poisoned.into_inner())
2066            .insert(module_id.to_string(), Some(nonce));
2067    }
2068
2069    /// Record namespace prefixes owned by a supervised module.
2070    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2071        let mut owners = self
2072            .reserved_prefix_owners
2073            .lock()
2074            .unwrap_or_else(|poisoned| poisoned.into_inner());
2075        owners.retain(|_, owner| owner != owner_module_id);
2076        for prefix in prefixes {
2077            owners.insert(prefix.clone(), owner_module_id.to_string());
2078        }
2079    }
2080
2081    /// The launch nonce most recently minted for a module's spawn, if any.
2082    #[cfg(test)]
2083    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2084        self.spawn_nonces
2085            .lock()
2086            .unwrap_or_else(|poisoned| poisoned.into_inner())
2087            .get(module_id)
2088            .cloned()
2089    }
2090
2091    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2092        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2093        let spawn_nonce = self
2094            .spawn_nonces
2095            .lock()
2096            .unwrap_or_else(|poisoned| poisoned.into_inner())
2097            .get(&spec.module_id)
2098            .cloned();
2099        let mut reserved_nonces = self
2100            .reserved_nonces
2101            .lock()
2102            .unwrap_or_else(|poisoned| poisoned.into_inner());
2103        if spec.reserved {
2104            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2105            // reserved name whose module has never spawned has no legitimate
2106            // holder, and the entry's absence is what used to leave the name
2107            // open to the first claimant.
2108            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2109        }
2110        drop(reserved_nonces);
2111        // A later unreserved declaration must not silently unreserve an id that
2112        // was retained after its reserved configuration was removed. The explicit
2113        // release ceremony is the only operation that retires that gate.
2114        self.removal_tombstones
2115            .lock()
2116            .unwrap_or_else(|poisoned| poisoned.into_inner())
2117            .remove(&spec.module_id);
2118    }
2119
2120    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2121    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2122    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2123    /// with no matching prefix are always authorized.
2124    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2125        self.reserved_hello_rejection(module_id, presented)
2126            .is_none()
2127    }
2128
2129    pub(crate) fn reserved_hello_rejection(
2130        &self,
2131        module_id: &str,
2132        presented: Option<&str>,
2133    ) -> Option<ReservedHelloRejection> {
2134        let nonces = self
2135            .reserved_nonces
2136            .lock()
2137            .unwrap_or_else(|poisoned| poisoned.into_inner());
2138        if let Some(expected) = nonces.get(module_id) {
2139            // `None` = reserved with no legitimate holder: refuse every
2140            // presentation, because no process can hold a nonce that was never
2141            // minted. Only a real minted nonce admits, in constant time.
2142            let authorized = match expected {
2143                Some(expected) => {
2144                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2145                }
2146                None => false,
2147            };
2148            if authorized {
2149                return None;
2150            }
2151            return Some(ReservedHelloRejection::Exact {
2152                module_id: module_id.to_string(),
2153            });
2154        }
2155        drop(nonces);
2156
2157        let matched_prefix = self
2158            .reserved_prefix_owners
2159            .lock()
2160            .unwrap_or_else(|poisoned| poisoned.into_inner())
2161            .iter()
2162            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2163            .max_by_key(|(prefix, _)| prefix.len())
2164            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2165        let (prefix, owner_module_id) = matched_prefix?;
2166
2167        let authorized = presented.is_some_and(|presented| {
2168            self.spawn_nonces
2169                .lock()
2170                .unwrap_or_else(|poisoned| poisoned.into_inner())
2171                .get(&owner_module_id)
2172                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2173                // While the owner is being swapped, children started by
2174                // either of its two processes hold that process's nonce.
2175                || self.swap_nonce_matches(&owner_module_id, presented)
2176        });
2177        if authorized {
2178            None
2179        } else {
2180            Some(ReservedHelloRejection::Prefix {
2181                prefix,
2182                owner_module_id,
2183            })
2184        }
2185    }
2186
2187    /// Whether a consumer connection proved it came from a daemon-spawned module.
2188    ///
2189    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2190    /// accepted only for module ids the supervisor has spawned.
2191    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2192        if presented.is_empty() {
2193            return false;
2194        }
2195        let nonces = self
2196            .spawn_nonces
2197            .lock()
2198            .unwrap_or_else(|poisoned| poisoned.into_inner());
2199        let current = nonces
2200            .get(module_id)
2201            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2202        drop(nonces);
2203        // During a swap two processes of the module are alive, and a consumer
2204        // started by either one presents that process's nonce. Accepting only
2205        // the recorded one would fail the incumbent's consumers for the whole
2206        // overlap once cutover moves the record to the candidate.
2207        current || self.swap_nonce_matches(module_id, presented)
2208    }
2209
2210    /// Whether `presented` is either nonce of an open swap for `module_id`.
2211    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2212        let swaps = self
2213            .swaps
2214            .lock()
2215            .unwrap_or_else(|poisoned| poisoned.into_inner());
2216        swaps.get(module_id).is_some_and(|swap| {
2217            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2218                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2219                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2220                })
2221        })
2222    }
2223
2224    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2225    /// Called before the candidate process exists.
2226    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2227        let incumbent_nonce = self
2228            .spawn_nonces
2229            .lock()
2230            .unwrap_or_else(|poisoned| poisoned.into_inner())
2231            .get(module_id)
2232            .cloned();
2233        self.swaps
2234            .lock()
2235            .unwrap_or_else(|poisoned| poisoned.into_inner())
2236            .insert(
2237                module_id.to_string(),
2238                OpenSwap {
2239                    candidate_nonce,
2240                    incumbent_nonce,
2241                    candidate_admitted: false,
2242                },
2243            );
2244    }
2245
2246    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2247    /// the module's recorded one.
2248    pub(crate) fn close_swap(&self, module_id: &str) {
2249        self.swaps
2250            .lock()
2251            .unwrap_or_else(|poisoned| poisoned.into_inner())
2252            .remove(module_id);
2253    }
2254
2255    /// Install the observer told about swap promotions, replacing any earlier
2256    /// one.
2257    pub(crate) fn set_swap_promotion_observer(
2258        &self,
2259        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2260    ) {
2261        *self
2262            .promotion_observer
2263            .0
2264            .lock()
2265            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2266    }
2267
2268    /// Tell the installed observer, if it is still alive, that a swap promoted
2269    /// `registration`.
2270    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2271        let observer = self
2272            .promotion_observer
2273            .0
2274            .lock()
2275            .unwrap_or_else(|poisoned| poisoned.into_inner())
2276            .as_ref()
2277            .and_then(std::sync::Weak::upgrade);
2278        if let Some(observer) = observer {
2279            observer.swap_promoted(registration);
2280        }
2281    }
2282
2283    /// Whether a swap is open for `module_id`.
2284    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2285        self.swaps
2286            .lock()
2287            .unwrap_or_else(|poisoned| poisoned.into_inner())
2288            .contains_key(module_id)
2289    }
2290
2291    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2292    /// respawn would, once cutover has made the candidate the module's process.
2293    /// The swap stays open so the incumbent's nonce keeps attesting until the
2294    /// incumbent has drained and exited.
2295    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2296        let candidate_nonce = self
2297            .swaps
2298            .lock()
2299            .unwrap_or_else(|poisoned| poisoned.into_inner())
2300            .get(module_id)
2301            .map(|swap| swap.candidate_nonce.clone());
2302        let Some(nonce) = candidate_nonce else {
2303            return;
2304        };
2305        self.set_spawn_nonce(module_id, nonce.clone());
2306        if reserved {
2307            self.set_reserved_nonce(module_id, nonce);
2308        }
2309    }
2310
2311    /// The swap gate for a HELLO claiming `module_id`.
2312    ///
2313    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2314    /// presents the candidate nonce, which the reserved gate (holding the
2315    /// incumbent's nonce) would refuse as `reserved_module` before swap
2316    /// admission was ever reached. And it applies to unreserved ids too: for an
2317    /// unreserved id the only thing that ever stopped a second process claiming
2318    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2319    /// refusal a swap lifts for its candidate.
2320    ///
2321    /// The incumbent's own nonce falls through to the ordinary gates, which
2322    /// treat it as they always have (a live incumbent is refused as a
2323    /// duplicate). Anything else while a swap is open is refused, including an
2324    /// absent nonce.
2325    pub(crate) fn swap_hello_admission(
2326        &self,
2327        module_id: &str,
2328        presented: Option<&str>,
2329    ) -> SwapHelloAdmission {
2330        let swaps = self
2331            .swaps
2332            .lock()
2333            .unwrap_or_else(|poisoned| poisoned.into_inner());
2334        let Some(swap) = swaps.get(module_id) else {
2335            return SwapHelloAdmission::NotSwapping;
2336        };
2337        let Some(presented) = presented else {
2338            return SwapHelloAdmission::Refused;
2339        };
2340        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2341            return if swap.candidate_admitted {
2342                SwapHelloAdmission::Refused
2343            } else {
2344                SwapHelloAdmission::Candidate
2345            };
2346        }
2347        if swap
2348            .incumbent_nonce
2349            .as_deref()
2350            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2351        {
2352            return SwapHelloAdmission::NotSwapping;
2353        }
2354        SwapHelloAdmission::Refused
2355    }
2356
2357    /// Record that the swap token has registered a candidate, so it admits no
2358    /// second HELLO.
2359    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2360        if let Some(swap) = self
2361            .swaps
2362            .lock()
2363            .unwrap_or_else(|poisoned| poisoned.into_inner())
2364            .get_mut(module_id)
2365        {
2366            swap.candidate_admitted = true;
2367        }
2368    }
2369
2370    /// Test/support lookup for the current launch nonce of a supervised spawn.
2371    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2372        self.spawn_nonces
2373            .lock()
2374            .unwrap_or_else(|poisoned| poisoned.into_inner())
2375            .get(module_id)
2376            .cloned()
2377    }
2378
2379    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2380    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2381        self.reserved_nonces
2382            .lock()
2383            .unwrap_or_else(|poisoned| poisoned.into_inner())
2384            .get(module_id)
2385            .cloned()
2386            .flatten()
2387    }
2388
2389    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2390        // Normally already marked before the process was spawned; marking here
2391        // too keeps `configured_ids` a superset of the roster for any caller
2392        // that inserts a module directly.
2393        self.mark_configured(module.module_id());
2394        let mut modules = self
2395            .modules
2396            .lock()
2397            .unwrap_or_else(|poisoned| poisoned.into_inner());
2398        modules.insert(module.module_id().to_string(), module)
2399    }
2400
2401    /// Record that the supervisor has taken on `module_id`. Called before the
2402    /// module's first process is spawned, so that by the time that process can
2403    /// register, [`Self::is_configured`] already answers true.
2404    fn mark_configured(&self, module_id: &str) {
2405        self.configured_ids
2406            .lock()
2407            .unwrap_or_else(|poisoned| poisoned.into_inner())
2408            .insert(module_id.to_string());
2409    }
2410
2411    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2412    /// before it was ever put on the roster. A module already on the roster
2413    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2414    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2415        let modules = self
2416            .modules
2417            .lock()
2418            .unwrap_or_else(|poisoned| poisoned.into_inner());
2419        if !modules.contains_key(module_id) {
2420            self.configured_ids
2421                .lock()
2422                .unwrap_or_else(|poisoned| poisoned.into_inner())
2423                .remove(module_id);
2424        }
2425    }
2426
2427    /// Whether `module_id` is a module this daemon supervises: on the roster,
2428    /// or about to be (its process is being spawned right now).
2429    ///
2430    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2431    /// for scopes: a supervised module's process can register and sync before
2432    /// [`Self::get`] can return it, and in that window it is still a module
2433    /// that will sync, not one that never will.
2434    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2435        self.configured_ids
2436            .lock()
2437            .unwrap_or_else(|poisoned| poisoned.into_inner())
2438            .contains(module_id)
2439    }
2440
2441    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2442        let modules = self
2443            .modules
2444            .lock()
2445            .unwrap_or_else(|poisoned| poisoned.into_inner());
2446        modules.get(module_id).cloned()
2447    }
2448
2449    pub(crate) fn record_late_health_answer(
2450        &self,
2451        module_id: &str,
2452        latency_ms: u64,
2453    ) -> Result<bool, SuperviseError> {
2454        let Some(module) = self.get(module_id) else {
2455            return Ok(false);
2456        };
2457        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2458            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2459            state.health.last_late_answer_latency_ms = Some(latency_ms);
2460            // A late answer is an answer: the module served the probe, just past
2461            // the deadline. Leaving the miss streak in place while logging
2462            // "proves the module is alive" is how a CPU-starved module that
2463            // answers every probe a few seconds late still marches to the
2464            // threshold and gets killed — the exact kill class `NoAnswer` is
2465            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2466            // is degradation, and degradation reports; it does not restart.
2467            state.health.consecutive_failures = 0;
2468        })?;
2469        Ok(true)
2470    }
2471
2472    /// Arm the one-shot marker for the module process that this caller
2473    /// deliberately initiated severance against. Generic connection teardown
2474    /// must not call this:
2475    /// a surviving process would otherwise retain an exemption for a later
2476    /// genuine crash.
2477    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2478        let Some(module) = self.get(module_id) else {
2479            return Ok(false);
2480        };
2481        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2482        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2483            return Ok(false);
2484        };
2485        drop(snapshot);
2486        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2487    }
2488
2489    pub fn list(&self) -> Vec<SupervisedModule> {
2490        let modules = self
2491            .modules
2492            .lock()
2493            .unwrap_or_else(|poisoned| poisoned.into_inner());
2494        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2495        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2496        modules
2497    }
2498
2499    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2500        self.activity_images
2501            .lock()
2502            .unwrap_or_else(|p| p.into_inner())
2503            .remove(module_id);
2504        self.spawn_nonces
2505            .lock()
2506            .unwrap_or_else(|poisoned| poisoned.into_inner())
2507            .remove(module_id);
2508        self.close_swap(module_id);
2509        let mut reserved_nonces = self
2510            .reserved_nonces
2511            .lock()
2512            .unwrap_or_else(|poisoned| poisoned.into_inner());
2513        if reserved_nonces.contains_key(module_id) {
2514            // The old nonce must die with the removed process, but the exact-id
2515            // gate remains until an operator explicitly releases it.
2516            reserved_nonces.insert(module_id.to_string(), None);
2517        }
2518        drop(reserved_nonces);
2519        self.reserved_prefix_owners
2520            .lock()
2521            .unwrap_or_else(|poisoned| poisoned.into_inner())
2522            .retain(|_, owner| owner != module_id);
2523        let removed = self
2524            .modules
2525            .lock()
2526            .unwrap_or_else(|poisoned| poisoned.into_inner())
2527            .remove(module_id);
2528        self.configured_ids
2529            .lock()
2530            .unwrap_or_else(|poisoned| poisoned.into_inner())
2531            .remove(module_id);
2532        removed
2533    }
2534
2535    /// Remember a module removed by a non-preview rescan so route.open can
2536    /// distinguish that intentional removal from an unknown id.
2537    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2538        self.removal_tombstones
2539            .lock()
2540            .unwrap_or_else(|poisoned| poisoned.into_inner())
2541            .insert(module_id.to_string(), unix_ms_now());
2542    }
2543
2544    /// Return how long ago a rescan removed this module in milliseconds.
2545    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2546        self.removal_tombstones
2547            .lock()
2548            .unwrap_or_else(|poisoned| poisoned.into_inner())
2549            .get(module_id)
2550            .copied()
2551            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2552    }
2553
2554    /// Retire a reserved-id gate only after its module has left supervision.
2555    ///
2556    /// A retained gate has no live nonce (`None`), so releasing any other entry
2557    /// would weaken a currently configured or otherwise active reservation.
2558    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2559        if self.get(module_id).is_some() {
2560            return false;
2561        }
2562        let mut reserved_nonces = self
2563            .reserved_nonces
2564            .lock()
2565            .unwrap_or_else(|poisoned| poisoned.into_inner());
2566        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2567            return false;
2568        }
2569        reserved_nonces.remove(module_id);
2570        true
2571    }
2572
2573    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2574        Arc::clone(&self.operation_lock)
2575    }
2576}
2577
2578/// Process supervisor for subc-owned singleton modules.
2579#[derive(Debug, Clone)]
2580pub struct Supervisor {
2581    registry: Arc<Registry>,
2582    restart_policy: RestartPolicy,
2583    drain_timeout: Duration,
2584    connection_file_path: Option<PathBuf>,
2585    capture_logs_dir: Option<PathBuf>,
2586    forwarding: Option<Arc<ForwardingTable>>,
2587    process_liveness: Arc<SupervisorProcessLiveness>,
2588    supervisor_handle: Option<SupervisorHandle>,
2589    health: HealthConfig,
2590    daemon_start_clock: crate::clock::StartClock,
2591    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2592    spawn_events: SpawnEventFeed,
2593    provenance_probe: ExecutableIdentityProbe,
2594    /// Every process spawned through this supervisor (and its clones) and not
2595    /// yet reaped, so daemon shutdown can end them.
2596    child_roster: ChildRoster,
2597    #[cfg(target_os = "linux")]
2598    cgroup_placement: Option<subc_cgroup::Placement>,
2599    #[cfg(test)]
2600    test_after_first_spawn: AfterFirstSpawnHook,
2601}
2602
2603/// A roster entry whose monitor and first process have not started yet.
2604/// Keeping the command receiver here lets discovery serve status immediately
2605/// while start/stop requests wait until bootstrap has published the connection file.
2606pub(crate) struct RegisteredConfiguredModule {
2607    spec: ModuleSpec,
2608    runtime: SupervisorRuntimeConfig,
2609    module: SupervisedModule,
2610    commands: mpsc::Receiver<SupervisorCommand>,
2611}
2612
2613/// Test-only hook run on the path that takes on a new module, right after its
2614/// first `spawn_child` returns (the process exists and could already be
2615/// registering) and before that process is handed to the module's supervise
2616/// loop and put on the roster. Lets a test observe what a fast child would see
2617/// in that window without racing a real one.
2618#[cfg(test)]
2619#[derive(Clone, Default)]
2620struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2621
2622#[cfg(test)]
2623type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2624
2625#[cfg(test)]
2626impl fmt::Debug for AfterFirstSpawnHook {
2627    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2628        f.write_str("AfterFirstSpawnHook")
2629    }
2630}
2631
2632#[cfg(test)]
2633impl AfterFirstSpawnHook {
2634    fn run(&self, module_id: &str) {
2635        if let Some(hook) = &self.0 {
2636            hook(module_id);
2637        }
2638    }
2639}
2640
2641impl Supervisor {
2642    #[cfg(test)]
2643    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2644        let supervisor = Self::new(registry, policy);
2645        #[cfg(target_os = "macos")]
2646        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2647        supervisor
2648    }
2649    /// Verify the trampoline once when configured. Missing private OS support
2650    /// refuses every macOS launch by name but does not stop the daemon's control
2651    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2652    /// entry point; the library must not exec an arbitrary hosting program.
2653    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2654        let path = path.into();
2655        #[cfg(target_os = "macos")]
2656        {
2657            let result = probe_privacy_trampoline(&path).map(|()| path);
2658            if let Err(cause) = &result {
2659                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2660            }
2661            self.child_roster.set_privacy_trampoline(result);
2662        }
2663        #[cfg(not(target_os = "macos"))]
2664        let _ = path;
2665        self
2666    }
2667    /// The first step of an announced daemon shutdown, before the notice and
2668    /// before any connection is closed.
2669    ///
2670    /// Sets the daemon-shutdown flag first: from here on no module is
2671    /// respawned (crash restart, operator restart, or swap), and every child
2672    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2673    /// the module exits on the EOF this shutdown gives it or is signalled by a
2674    /// service manager that kills the whole cgroup. Then writes the journal's
2675    /// shutdown marker, which records the instant and closes this daemon
2676    /// incarnation's stretch of the journal.
2677    #[cfg(unix)]
2678    pub(crate) fn begin_daemon_shutdown(&self) {
2679        self.child_roster.close();
2680        if let Some(journal) = &self.terminal_journal {
2681            journal.stamp_shutdown();
2682        }
2683    }
2684
2685    /// Announce a cut while established connections can still carry replies.
2686    /// These budgets promise notice and a bounded wait, not child completion;
2687    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2688    #[cfg(unix)]
2689    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2690        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2691        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2692        let Some(forwarding) = &self.forwarding else {
2693            return Ok(());
2694        };
2695        let modules = forwarding
2696            .begin_daemon_drain()
2697            .map_err(SuperviseError::Forwarding)?;
2698        let deadline_ms =
2699            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2700        let mut notices = tokio::task::JoinSet::new();
2701        let mut drains = Vec::new();
2702        for (module_id, endpoint) in modules {
2703            let Some(target) = forwarding
2704                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2705                .map_err(SuperviseError::Forwarding)?
2706            else {
2707                continue;
2708            };
2709            let routes = forwarding
2710                .endpoint_routes(target.endpoint)
2711                .map_err(SuperviseError::Forwarding)?;
2712            // Restart allows deployed consumers to reopen after the new daemon
2713            // appears. The wire reason stays `restart`; what tells a daemon cut
2714            // apart from a module restart afterwards is the terminal record
2715            // itself, whose disposition is `daemon_shutdown` for every exit
2716            // observed once `begin_daemon_shutdown` has run.
2717            let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2718                reason: RouteCloseReason::Restart,
2719                deadline_ms,
2720            })
2721            .expect("module draining serializes");
2722            let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2723            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2724            for route in routes {
2725                let client = route.goodbye_target;
2726                if let Some((_, channels)) = clients
2727                    .iter_mut()
2728                    .find(|(existing, _)| existing.connection_id == client.connection_id)
2729                {
2730                    channels.push(client.channel);
2731                } else {
2732                    let channel = client.channel;
2733                    clients.push((client, vec![channel]));
2734                }
2735            }
2736            for (client, mut channels) in clients {
2737                channels.sort_unstable();
2738                channels.dedup();
2739                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2740                    module_id: module_id.clone(),
2741                    channels,
2742                    reason: RouteCloseReason::Restart,
2743                })
2744                .expect("route closing serializes");
2745                recipients.push((client.sink, client.negotiated_ver, closing));
2746            }
2747            for (sink, version, body) in recipients {
2748                notices.spawn(async move {
2749                    let frame = Frame::build_with_version(
2750                        version,
2751                        FrameType::Push,
2752                        control_flags(),
2753                        0,
2754                        0,
2755                        0,
2756                        body,
2757                    )
2758                    .expect("bounded lifecycle notice frame builds");
2759                    sink.send_flushed(frame).await
2760                });
2761            }
2762            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2763            drains.push((module_id, target.endpoint, gauges));
2764        }
2765        // A quiet forwarding table is not proof that queued notices reached the
2766        // socket. Wait for writer flush acknowledgements before testing quiescence.
2767        let notice_deadline = Instant::now() + NOTICE_BUDGET;
2768        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2769            if !matches!(result, Ok(Ok(()))) {
2770                warn!(?result, "daemon shutdown notice delivery failed");
2771            }
2772        }
2773        notices.abort_all();
2774        let deadline = Instant::now() + DRAIN_BUDGET;
2775        let mut waits = tokio::task::JoinSet::new();
2776        for (module_id, endpoint, gauges) in drains {
2777            let forwarding = Arc::clone(forwarding);
2778            let mut runtime = self.runtime_config();
2779            runtime.health.cadence = Duration::from_millis(100);
2780            waits.spawn(async move {
2781                let wait_result = wait_for_forwarding_quiescence(
2782                    &forwarding,
2783                    &module_id,
2784                    &runtime,
2785                    endpoint,
2786                    deadline,
2787                    &gauges,
2788                    DrainScope::Endpoint(endpoint),
2789                )
2790                .await;
2791                // Store the measurement before the task returns: quiescence
2792                // may be reached just as the deadline stops the join loop from
2793                // collecting results. Unfinished or failed waits report false.
2794                forwarding
2795                    .record_daemon_drain_result(
2796                        endpoint,
2797                        drained_after_quiescence_wait(&wait_result),
2798                    )
2799                    .map_err(SuperviseError::Forwarding)?;
2800                wait_result
2801            });
2802        }
2803        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2804            if !matches!(result, Ok(Ok(true))) {
2805                warn!(?result, "daemon shutdown drain did not reach quiescence");
2806            }
2807        }
2808        Ok(())
2809    }
2810
2811    /// The last step of an announced daemon shutdown, after the notice and the
2812    /// drain: send every registered module a module GOODBYE, the same planned
2813    /// stop signal `ck module stop` gives, then close every connection so each
2814    /// subc module sees EOF and starts its own teardown, then end every
2815    /// supervised child that has not exited
2816    /// by its own deadline (its drain budget, capped). Modules lead their own
2817    /// process groups, so a
2818    /// service manager's group kill no longer reaches them; without this a
2819    /// child that does not stop on EOF (every `protocol: "none"` child, which
2820    /// has no connection) would outlive the daemon. Every wait is bounded (see
2821    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
2822    #[cfg(unix)]
2823    pub(crate) async fn end_children_for_daemon_shutdown(
2824        &self,
2825        already_escalated: bool,
2826        escalate: impl std::future::Future<Output = ()>,
2827    ) {
2828        tokio::pin!(escalate);
2829        let mut escalated = already_escalated;
2830        if let Some(forwarding) = &self.forwarding {
2831            let reason = CloseReason::new(
2832                "daemon_shutdown",
2833                "the daemon is exiting after its shutdown notice and drain",
2834            );
2835            if escalated {
2836                // The operator asked to stop waiting: queue the GOODBYEs but
2837                // do not wait for them to be written.
2838                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2839            } else {
2840                tokio::select! {
2841                    biased;
2842                    _ = escalate.as_mut() => {
2843                        info!("second SIGTERM: abandoning module GOODBYE delivery");
2844                        escalated = true;
2845                    }
2846                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2847                }
2848            }
2849            let closed = forwarding.close_all_connections(&reason);
2850            debug!(closed, "closed established connections for daemon shutdown");
2851        }
2852        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
2853        // already completed and must not be polled again; the child shutdown
2854        // wait is told it is escalated and gets a future that never fires.
2855        let escalated_here = escalated && !already_escalated;
2856        let remaining_escalate = async move {
2857            if escalated_here {
2858                std::future::pending::<()>().await;
2859            } else {
2860                escalate.await;
2861            }
2862        };
2863        crate::child_roster::end_children_for_daemon_shutdown(
2864            &self.child_roster,
2865            escalated,
2866            remaining_escalate,
2867        )
2868        .await;
2869    }
2870
2871    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2872        Self {
2873            registry,
2874            restart_policy,
2875            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2876            connection_file_path: None,
2877            capture_logs_dir: None,
2878            forwarding: None,
2879            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2880            supervisor_handle: None,
2881            health: HealthConfig::default(),
2882            daemon_start_clock: crate::clock::StartClock::capture(),
2883            terminal_journal: None,
2884            spawn_events: SpawnEventFeed::default(),
2885            provenance_probe: ExecutableIdentityProbe::default(),
2886            child_roster: ChildRoster::default(),
2887            #[cfg(target_os = "linux")]
2888            cgroup_placement: None,
2889            #[cfg(test)]
2890            test_after_first_spawn: AfterFirstSpawnHook::default(),
2891        }
2892    }
2893
2894    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
2895        self.drain_timeout = drain_timeout;
2896        self
2897    }
2898
2899    pub fn with_process_liveness(
2900        mut self,
2901        process_liveness: Arc<SupervisorProcessLiveness>,
2902    ) -> Self {
2903        self.process_liveness = process_liveness;
2904        self
2905    }
2906
2907    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
2908        self.connection_file_path = Some(connection_file_path.into());
2909        self
2910    }
2911
2912    /// Enables daemon-owned capture files for supervised stdout and stderr.
2913    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
2914        self.capture_logs_dir = Some(logs_dir.into());
2915        self
2916    }
2917
2918    /// Names this daemon lifetime in spawn events, independently of whether a
2919    /// terminal journal is configured.
2920    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
2921        // A millisecond start stamp can repeat after clock rollback or a rapid
2922        // restart. Use the connection file's random daemon_id instead: it already
2923        // identifies this daemon lifetime independently of the wall clock.
2924        self.spawn_events.configure_incarnation(daemon_incarnation);
2925        self
2926    }
2927
2928    /// Enables best-effort history shared by every supervised module. Without
2929    /// it, terminal history is kept only in each module's in-memory ring.
2930    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
2931        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
2932        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
2933            path,
2934            daemon_incarnation,
2935        )));
2936        this
2937    }
2938
2939    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
2940        self.forwarding = Some(forwarding);
2941        self
2942    }
2943
2944    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
2945        self.spawn_events = supervisor_handle.spawn_events.clone();
2946        self.supervisor_handle = Some(supervisor_handle);
2947        self
2948    }
2949
2950    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
2951        self.health = health;
2952        self
2953    }
2954
2955    /// Keeps a record of every live child at `path`, rewritten on each spawn and
2956    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
2957    /// record is kept.
2958    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
2959        self.child_roster.record_to(path.into());
2960        self
2961    }
2962
2963    #[cfg(target_os = "linux")]
2964    pub fn with_cgroup_placement(
2965        mut self,
2966        cgroup_placement: Option<subc_cgroup::Placement>,
2967    ) -> Self {
2968        self.cgroup_placement = cgroup_placement;
2969        self
2970    }
2971
2972    /// Spawn `spec.program` and start monitoring it.
2973    ///
2974    /// The child is expected to parse `--subc <connection-file-path>`, read the
2975    /// TCP+key connection file, authenticate to the already-running listener, and
2976    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
2977    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
2978        validate_spec(&spec)?;
2979        self.establish_identity(&spec);
2980
2981        let runtime = self.runtime_config();
2982        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
2983        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
2984        let spawned = spawn_child(
2985            &spec,
2986            runtime.connection_file_path.as_deref(),
2987            self.supervisor_handle.as_ref(),
2988            &runtime.stderr_ring,
2989            runtime.capture_logs_dir.as_deref(),
2990            &runtime.child_roster,
2991            #[cfg(target_os = "linux")]
2992            runtime.cgroup_placement.as_ref(),
2993        );
2994        #[cfg(test)]
2995        self.test_after_first_spawn.run(&spec.module_id);
2996        let child = match spawned {
2997            Ok(child) => child,
2998            Err(err) => {
2999                // Unlike the configured paths, a failed `spawn` leaves nothing
3000                // on the roster, so the module must not stay marked configured.
3001                self.abandon_unrostered(&spec.module_id);
3002                return Err(err);
3003            }
3004        };
3005        self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3006        if let Some(spawn) = activity_spawn {
3007            spawn.complete(&snapshot);
3008        }
3009
3010        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3011    }
3012
3013    /// Make `spec`'s module count as configured, with its identity gates
3014    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3015    /// exists.
3016    ///
3017    /// Every path that takes on a new module calls this before `spawn_child`.
3018    /// The order is the point: the child can connect, register, sync its
3019    /// scopes and ask about them as soon as it is spawned, and the module is
3020    /// only put on the roster after `spawn_child` returns. Were the mark set
3021    /// with the roster entry, a fast child would see its own owner reported
3022    /// as not configured, and a scoped `route.open` in that window would be
3023    /// refused as terminal `scope_not_live` ("will never sync") instead of
3024    /// retryable `scope_not_synced`.
3025    fn establish_identity(&self, spec: &ModuleSpec) {
3026        if let Some(supervisor_handle) = &self.supervisor_handle {
3027            supervisor_handle.apply_identity_configuration(spec);
3028            supervisor_handle.mark_configured(&spec.module_id);
3029        }
3030    }
3031
3032    /// Take back [`Self::establish_identity`]'s configured mark when the
3033    /// module will not be put on the roster after all.
3034    fn abandon_unrostered(&self, module_id: &str) {
3035        if let Some(supervisor_handle) = &self.supervisor_handle {
3036            supervisor_handle.unmark_configured_unless_rostered(module_id);
3037        }
3038    }
3039
3040    /// Record a freshly spawned first process as running. On failure the
3041    /// module never reaches the roster, so its configured mark is taken back.
3042    fn mark_first_process_running(
3043        &self,
3044        spec: &ModuleSpec,
3045        runtime: &SupervisorRuntimeConfig,
3046        snapshot: &SharedSnapshot,
3047        child: &SupervisedChild,
3048    ) -> Result<(), SuperviseError> {
3049        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3050            self.abandon_unrostered(&spec.module_id);
3051            return Err(err);
3052        }
3053        self.process_liveness
3054            .track(spec.module_id.clone(), Arc::clone(snapshot));
3055        Ok(())
3056    }
3057
3058    /// Start supervising a module declared in daemon configuration.
3059    ///
3060    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3061    /// failures in the supervisor handle so operator-facing `supervisor.list`
3062    /// reflects every configured module while daemon startup continues.
3063    pub fn supervise_configured(
3064        &self,
3065        spec: ModuleSpec,
3066        enabled: bool,
3067    ) -> Result<SupervisedModule, SuperviseError> {
3068        self.supervise_configured_requested(
3069            spec,
3070            enabled,
3071            self.health.clone(),
3072            None,
3073            self.restart_policy,
3074            None,
3075        )
3076    }
3077
3078    /// Supervise a configured module with its own health, drain, and crash
3079    /// budget. The restart policy is per-module because the config file is:
3080    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3081    /// module that is expensive to restart should not be forced onto the same
3082    /// budget as one that is cheap.
3083    pub fn supervise_configured_with_health(
3084        &self,
3085        spec: ModuleSpec,
3086        enabled: bool,
3087        health: HealthConfig,
3088        drain_timeout_ms: Option<u64>,
3089        restart_policy: RestartPolicy,
3090    ) -> Result<SupervisedModule, SuperviseError> {
3091        self.supervise_configured_requested(
3092            spec,
3093            enabled,
3094            health,
3095            drain_timeout_ms,
3096            restart_policy,
3097            None,
3098        )
3099    }
3100
3101    pub(crate) fn supervise_configured_requested(
3102        &self,
3103        spec: ModuleSpec,
3104        enabled: bool,
3105        health: HealthConfig,
3106        drain_timeout_ms: Option<u64>,
3107        restart_policy: RestartPolicy,
3108        requester: Option<Principal>,
3109    ) -> Result<SupervisedModule, SuperviseError> {
3110        let registered = self.register_configured_with_health(
3111            spec,
3112            enabled,
3113            health,
3114            drain_timeout_ms,
3115            restart_policy,
3116        )?;
3117        self.start_registered_configured(registered, requester)
3118    }
3119
3120    /// Record a configured module as starting or disabled, without launching a
3121    /// process or running its monitor. Bootstrap registers the entire roster
3122    /// before publishing discovery, then activates it once children can read
3123    /// the connection file passed in their `--subc` argument.
3124    pub(crate) fn register_configured_with_health(
3125        &self,
3126        spec: ModuleSpec,
3127        enabled: bool,
3128        health: HealthConfig,
3129        drain_timeout_ms: Option<u64>,
3130        restart_policy: RestartPolicy,
3131    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3132        validate_spec(&spec)?;
3133        self.establish_identity(&spec);
3134
3135        let mut runtime = self.runtime_config();
3136        runtime.health = health;
3137        runtime.restart_policy = restart_policy;
3138        if let Some(ms) = drain_timeout_ms {
3139            runtime.drain_timeout = Duration::from_millis(ms);
3140            *runtime
3141                .effective_drain_timeout
3142                .lock()
3143                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3144        }
3145        let snapshot = if enabled {
3146            SupervisorSnapshot::starting()
3147        } else {
3148            SupervisorSnapshot::disabled()
3149        };
3150        let snapshot = {
3151            let mut snapshot = snapshot;
3152            if enabled {
3153                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3154                {
3155                    snapshot.waiting_for_previous_pid = Some(pid);
3156                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3157                }
3158            }
3159            snapshot
3160        };
3161        let (module, commands) =
3162            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3163        Ok(RegisteredConfiguredModule {
3164            spec,
3165            runtime,
3166            module,
3167            commands,
3168        })
3169    }
3170
3171    /// Activate a previously registered module. Queued control requests are
3172    /// served by the monitor after the initial configured spawn has settled.
3173    pub(crate) fn start_registered_configured(
3174        &self,
3175        registered: RegisteredConfiguredModule,
3176        requester: Option<Principal>,
3177    ) -> Result<SupervisedModule, SuperviseError> {
3178        let RegisteredConfiguredModule {
3179            spec,
3180            mut runtime,
3181            module,
3182            commands,
3183        } = registered;
3184        runtime.start_requester = requester;
3185        let snapshot = &module.inner.snapshot;
3186        let should_spawn = {
3187            let state = lock_snapshot(snapshot)?;
3188            state.enabled && state.waiting_for_previous_pid.is_none()
3189        };
3190        let child = if should_spawn {
3191            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3192            let spawned = spawn_child(
3193                &spec,
3194                runtime.connection_file_path.as_deref(),
3195                self.supervisor_handle.as_ref(),
3196                &runtime.stderr_ring,
3197                runtime.capture_logs_dir.as_deref(),
3198                &runtime.child_roster,
3199                #[cfg(target_os = "linux")]
3200                runtime.cgroup_placement.as_ref(),
3201            );
3202            #[cfg(test)]
3203            self.test_after_first_spawn.run(&spec.module_id);
3204            match spawned {
3205                Ok(child) => {
3206                    self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3207                    if let Some(spawn) = activity_spawn {
3208                        spawn.complete(snapshot);
3209                    }
3210                    Some(child)
3211                }
3212                Err(err) => {
3213                    if runtime.health.critical {
3214                        error!(
3215                            module_id = %spec.module_id,
3216                            program = %spec.program.display(),
3217                            error = %err,
3218                            "critical configured module failed to spawn; marking failed and alerting"
3219                        );
3220                    } else {
3221                        error!(
3222                            module_id = %spec.module_id,
3223                            program = %spec.program.display(),
3224                            error = %err,
3225                            "configured module failed to spawn; marking failed and continuing"
3226                        );
3227                    }
3228                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
3229                        state.state = ModuleState::Failed;
3230                        clear_current_process_facts(state);
3231                    })?;
3232                    None
3233                }
3234            }
3235        } else {
3236            None
3237        };
3238        self.start_monitor(spec, runtime, &module, child, commands);
3239        Ok(module)
3240    }
3241
3242    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3243        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3244        SupervisorRuntimeConfig {
3245            start_requester: None,
3246            pending_activity_restart: false,
3247            provenance_probe: self.provenance_probe.clone(),
3248            scheduled_respawn: Arc::default(),
3249            deferred_reload_reply: Arc::default(),
3250            restart_policy: self.restart_policy,
3251            drain_timeout: self.drain_timeout,
3252            // Shared with this module's roster copy: daemon shutdown waits on
3253            // each child for the module's own drain budget, as resolved now.
3254            child_roster: self
3255                .child_roster
3256                .for_module(Arc::clone(&effective_drain_timeout)),
3257            effective_drain_timeout,
3258            default_drain_timeout: self.drain_timeout,
3259            health: self.health.clone(),
3260            connection_file_path: self.connection_file_path.clone(),
3261            capture_logs_dir: self.capture_logs_dir.clone(),
3262            forwarding: self.forwarding.clone(),
3263            supervisor_handle: self.supervisor_handle.clone(),
3264            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3265            terminal_ring: Arc::new(Mutex::new(
3266                TerminalRing::new(
3267                    TerminalRingConfig::default(),
3268                    self.daemon_start_clock.started_at_ms(),
3269                )
3270                .with_start_clock(self.daemon_start_clock)
3271                .with_journal(self.terminal_journal.clone())
3272                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3273            )),
3274            spawn_events: self.spawn_events.clone(),
3275            #[cfg(target_os = "linux")]
3276            cgroup_placement: self.cgroup_placement.clone(),
3277            #[cfg(test)]
3278            test_seed_stale_facts_before_enable_spawn: false,
3279            #[cfg(test)]
3280            test_reload_exit_record_gate: None,
3281        }
3282    }
3283
3284    fn supervised_module(
3285        &self,
3286        spec: ModuleSpec,
3287        runtime: SupervisorRuntimeConfig,
3288        snapshot: SharedSnapshot,
3289        child: Option<SupervisedChild>,
3290    ) -> SupervisedModule {
3291        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3292        self.start_monitor(spec, runtime, &module, child, commands);
3293        module
3294    }
3295
3296    fn register_module(
3297        &self,
3298        spec: &ModuleSpec,
3299        runtime: &SupervisorRuntimeConfig,
3300        snapshot: SharedSnapshot,
3301    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3302        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3303            spec: spec.clone(),
3304            health: runtime.health.clone(),
3305        }));
3306        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3307        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3308        // The module's OWN policy, which may be its per-module config rather than
3309        // the supervisor-wide one; status must report the budget the supervise
3310        // loop actually enforces.
3311        let restart_policy = runtime.restart_policy;
3312        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3313        let (tx, rx) = mpsc::channel(4);
3314        let module_id = spec.module_id.clone();
3315        let module = SupervisedModule {
3316            inner: Arc::new(SupervisedModuleInner {
3317                module_id: module_id.clone(),
3318                registry: Arc::clone(&self.registry),
3319                snapshot,
3320                configuration,
3321                stderr_ring,
3322                terminal_ring,
3323                commands: tx,
3324                monitor: Mutex::new(None),
3325                restart_policy,
3326                effective_drain_timeout,
3327                provenance_probe: self.provenance_probe.clone(),
3328            }),
3329        };
3330        // Identity gates and the configured mark must precede the roster entry,
3331        // so a client never sees a module without its admission policy.
3332        if let Some(supervisor_handle) = &self.supervisor_handle {
3333            supervisor_handle.insert(module.clone());
3334        }
3335        (module, rx)
3336    }
3337
3338    fn start_monitor(
3339        &self,
3340        spec: ModuleSpec,
3341        runtime: SupervisorRuntimeConfig,
3342        module: &SupervisedModule,
3343        child: Option<SupervisedChild>,
3344        commands: mpsc::Receiver<SupervisorCommand>,
3345    ) {
3346        let monitor = tokio::spawn(supervise_loop(
3347            spec,
3348            runtime,
3349            Arc::clone(&self.registry),
3350            Arc::clone(&self.process_liveness),
3351            Arc::clone(&module.inner.snapshot),
3352            child,
3353            commands,
3354        ));
3355        *module
3356            .inner
3357            .monitor
3358            .lock()
3359            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3360    }
3361}
3362
3363impl Default for Supervisor {
3364    fn default() -> Self {
3365        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3366    }
3367}
3368
3369/// Handle to one supervised child process.
3370#[derive(Clone)]
3371pub struct SupervisedModule {
3372    inner: Arc<SupervisedModuleInner>,
3373}
3374
3375struct SupervisedModuleInner {
3376    module_id: String,
3377    registry: Arc<Registry>,
3378    snapshot: SharedSnapshot,
3379    configuration: Arc<Mutex<SupervisedConfiguration>>,
3380    stderr_ring: Arc<Mutex<StderrRing>>,
3381    terminal_ring: Arc<Mutex<TerminalRing>>,
3382    commands: mpsc::Sender<SupervisorCommand>,
3383    monitor: Mutex<Option<JoinHandle<()>>>,
3384    /// Copied from the supervisor's runtime config at spawn so `status()` can
3385    /// report the restart budget without reaching back into the supervisor. The
3386    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3387    restart_policy: RestartPolicy,
3388    effective_drain_timeout: Arc<Mutex<Duration>>,
3389    provenance_probe: ExecutableIdentityProbe,
3390}
3391
3392impl fmt::Debug for SupervisedModule {
3393    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3394        f.debug_struct("SupervisedModule")
3395            .field("module_id", &self.inner.module_id)
3396            .field("status", &self.status())
3397            .finish_non_exhaustive()
3398    }
3399}
3400
3401impl SupervisedModule {
3402    pub fn module_id(&self) -> &str {
3403        &self.inner.module_id
3404    }
3405
3406    /// Test-only: put one probe miss on the streak, the way
3407    /// `handle_health_probe_failure` does, so tests can assert what a later
3408    /// event does to the streak without driving the whole probe loop.
3409    #[cfg(test)]
3410    pub(crate) fn record_health_probe_failure_for_test(
3411        &self,
3412        detail: &str,
3413    ) -> Result<(), SuperviseError> {
3414        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3415            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3416            state.health.detail = Some(detail.to_string());
3417        })
3418    }
3419
3420    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3421        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3422    }
3423
3424    pub(crate) fn registration_end_reason(
3425        &self,
3426    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3427        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3428        Ok(match snapshot.state {
3429            ModuleState::Draining if snapshot.draining_to_replace => {
3430                Some(RegistrationEndReason::SupervisorRestart)
3431            }
3432            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3433            ModuleState::Restarting if snapshot.draining_to_replace => {
3434                Some(RegistrationEndReason::SupervisorRestart)
3435            }
3436            _ => None,
3437        })
3438    }
3439
3440    /// The module's retained stderr, newest lines last.
3441    ///
3442    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3443    /// module, `supervisor.list` renders every module, and putting it in the
3444    /// shared snapshot would make each status read carry a payload almost nobody
3445    /// asked for. Callers that want the text ask for it.
3446    pub fn stderr_tail(
3447        &self,
3448        max_lines: Option<usize>,
3449        max_bytes: Option<usize>,
3450    ) -> StderrTailSnapshot {
3451        self.inner
3452            .stderr_ring
3453            .lock()
3454            .unwrap_or_else(|poisoned| poisoned.into_inner())
3455            .snapshot(max_lines, max_bytes)
3456    }
3457
3458    /// The module's bounded terminal history, oldest retained exit first.
3459    ///
3460    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3461    /// daemon whose in-memory history was necessarily reset.
3462    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3463        self.inner
3464            .terminal_ring
3465            .lock()
3466            .unwrap_or_else(|poisoned| poisoned.into_inner())
3467            .snapshot()
3468    }
3469
3470    /// Retained observations from the current ring and all journal generations.
3471    ///
3472    /// Blocking: this reads the journal files. Async callers use
3473    /// [`Self::read_durable_terminal_history`].
3474    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3475        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3476    }
3477
3478    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3479    /// read (up to every retained generation) never occupies a runtime worker.
3480    /// Fails only if the blocking task could not finish (runtime shutdown or a
3481    /// panic in the read).
3482    pub(crate) async fn read_durable_terminal_history(
3483        &self,
3484    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3485        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3486        let module_id = self.inner.module_id.clone();
3487        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3488            .await
3489    }
3490
3491    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3492        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3493            .map(|(status, _)| status)
3494    }
3495
3496    pub(crate) fn record_deliberate_severance(
3497        &self,
3498        identity: ProcessIdentity,
3499    ) -> Result<bool, SuperviseError> {
3500        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3501        if snapshot.pid != Some(identity.pid)
3502            || snapshot.process_start_time != Some(identity.start_time)
3503        {
3504            return Ok(false);
3505        }
3506        snapshot.deliberate_severance = Some(identity);
3507        Ok(true)
3508    }
3509
3510    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3511    ///
3512    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3513    /// does not produce reader-observability logs.
3514    pub(crate) fn status_for_control(
3515        &self,
3516        caller: &'static str,
3517    ) -> Result<ModuleStatus, SuperviseError> {
3518        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3519            .map(|(status, _)| status)
3520    }
3521
3522    fn status_with_snapshot_lock(
3523        &self,
3524        snapshot: &SharedSnapshot,
3525        caller: Option<&'static str>,
3526    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3527        let mut guard = match caller {
3528            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3529            None => lock_snapshot(snapshot)?,
3530        };
3531        // Read the budget through the pruning path so a reader sees the same
3532        // in-window count the restart decision would use, not a stale total.
3533        let restart_count =
3534            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3535        let snapshot = guard.clone();
3536        drop(guard);
3537        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3538            SuperviseError::StatePoisoned {
3539                module_id: Some(self.inner.module_id.clone()),
3540            }
3541        })?;
3542        let registration_active = self
3543            .inner
3544            .registry
3545            .get_module(&self.inner.module_id)
3546            .map_err(SuperviseError::Registry)?
3547            .is_some();
3548        let protocol = snapshot
3549            .spawned_protocol
3550            .unwrap_or(self.declared_protocol()?);
3551        let running_process =
3552            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3553        // Registration is the difference between the two protocols and the only
3554        // one: a subc module that has not registered cannot serve a request even
3555        // though its process is up, and a `none` module never registers at all,
3556        // so requiring it there would pin `live` to false for the whole life of
3557        // a perfectly healthy process.
3558        let live = match protocol {
3559            ModuleProtocol::Subc => running_process && registration_active,
3560            ModuleProtocol::None => running_process,
3561        };
3562
3563        Ok((
3564            ModuleStatus {
3565                waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3566                waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3567                module_id: self.inner.module_id.clone(),
3568                launch_nonce_source: observed_launch_nonce_source(&snapshot),
3569                state: snapshot.state,
3570                enabled: snapshot.enabled,
3571                process_alive: snapshot.process_alive,
3572                registration_active,
3573                protocol,
3574                live,
3575                restart_count,
3576                lifetime_restarts: snapshot.lifetime_restarts,
3577                spawn_generation: snapshot.spawn_generation,
3578                max_restarts: self.inner.restart_policy.max_restarts,
3579                restart_window: self.inner.restart_policy.window,
3580                drain_timeout,
3581                restart_backoff: self.inner.restart_policy.backoff,
3582                restart_max_backoff: self.inner.restart_policy.max_backoff,
3583                pid: snapshot.reported_pid(),
3584                spawned_at_ms: snapshot.spawned_at_ms,
3585                spawned_from: snapshot.spawned_from,
3586                process_start_time: snapshot.process_start_time,
3587                last_exit: snapshot.last_exit,
3588                health: snapshot.health,
3589            },
3590            snapshot.spawned_file_identity.clone(),
3591        ))
3592    }
3593
3594    #[cfg(test)]
3595    pub(crate) fn hold_snapshot_for_test(
3596        &self,
3597        acquired: std::sync::mpsc::Sender<()>,
3598        hold: Duration,
3599    ) -> std::thread::JoinHandle<()> {
3600        let snapshot = Arc::clone(&self.inner.snapshot);
3601        std::thread::spawn(move || {
3602            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3603            acquired
3604                .send(())
3605                .expect("test receiver waits for snapshot lock");
3606            std::thread::sleep(hold);
3607        })
3608    }
3609
3610    /// The status and the running-image check for `supervisor.provenance`,
3611    /// taken from one status read. The exec acknowledgement can land between
3612    /// two separate reads, and the reply would then pair "no pid yet" with an
3613    /// image observed after the module started, which describes no single
3614    /// moment.
3615    pub(crate) async fn status_and_running_image_agreement(
3616        &self,
3617    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3618        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3619        let image = self
3620            .inner
3621            .provenance_probe
3622            .observe(
3623                status.pid,
3624                status.spawned_from.as_deref(),
3625                identity,
3626                status.process_start_time,
3627            )
3628            .await;
3629        Ok((status, image))
3630    }
3631
3632    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3633        let snapshot = match lock_snapshot(&self.inner.snapshot) {
3634            Ok(snapshot) => snapshot.clone(),
3635            Err(_) => {
3636                return subc_control::RunningImageAgreement::Unavailable {
3637                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
3638                };
3639            }
3640        };
3641        self.inner
3642            .provenance_probe
3643            .observe(
3644                snapshot.reported_pid(),
3645                snapshot.spawned_from.as_deref(),
3646                snapshot.spawned_file_identity.clone(),
3647                snapshot.process_start_time,
3648            )
3649            .await
3650    }
3651
3652    /// Memory and CPU time of the module's current process, read now. Only the
3653    /// process the supervisor spawned is read, not processes it has started.
3654    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3655        #[cfg(windows)]
3656        {
3657            let image = lock_snapshot(&self.inner.snapshot)
3658                .ok()
3659                .and_then(|snapshot| {
3660                    snapshot
3661                        .spawned_file_identity
3662                        .as_ref()
3663                        .and_then(|identity| identity.windows_image.clone())
3664                });
3665            match image {
3666                Some(image) => {
3667                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
3668                }
3669                None => subc_control::ChildResourceUsage::Unavailable {
3670                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3671                },
3672            }
3673        }
3674        #[cfg(not(windows))]
3675        {
3676            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3677                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3678                Err(_) => {
3679                    return subc_control::ChildResourceUsage::Unavailable {
3680                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3681                    }
3682                }
3683            };
3684            crate::child_resources::read(pid, start_time)
3685        }
3686    }
3687
3688    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3689        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3690        Ok(match snapshot.state {
3691            ModuleState::Restarting => true,
3692            ModuleState::Failed | ModuleState::Disabled => false,
3693            _ if snapshot.enabled && snapshot.respawn_pending => true,
3694            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3695        })
3696    }
3697
3698    #[cfg(test)]
3699    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3700        self.is_warming_with_snapshot_lock(None)
3701    }
3702
3703    pub(crate) fn is_warming_for_control(
3704        &self,
3705        caller: &'static str,
3706    ) -> Result<bool, SuperviseError> {
3707        self.is_warming_with_snapshot_lock(Some(caller))
3708    }
3709
3710    fn is_warming_with_snapshot_lock(
3711        &self,
3712        caller: Option<&'static str>,
3713    ) -> Result<bool, SuperviseError> {
3714        let snapshot = match caller {
3715            Some(caller) => {
3716                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3717            }
3718            None => lock_snapshot(&self.inner.snapshot)?,
3719        }
3720        .clone();
3721        Ok(matches!(
3722            snapshot.state,
3723            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3724        ))
3725    }
3726
3727    /// Drain the module and stop monitoring it.
3728    pub async fn drain(&self) -> Result<(), SuperviseError> {
3729        self.stop().await
3730    }
3731
3732    pub(crate) async fn retire_requested(
3733        &self,
3734        requester: Option<Principal>,
3735    ) -> Result<(), SuperviseError> {
3736        match self.state()? {
3737            ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3738            ModuleState::Starting
3739            | ModuleState::Running
3740            | ModuleState::Unresponsive
3741            | ModuleState::Restarting
3742            | ModuleState::Draining
3743            | ModuleState::Disabled => {}
3744        }
3745
3746        let (reply_tx, reply_rx) = oneshot::channel();
3747        self.inner
3748            .commands
3749            .send(SupervisorCommand::Retire {
3750                requester,
3751                reply: reply_tx,
3752            })
3753            .await
3754            .map_err(|_| SuperviseError::CommandClosed {
3755                module_id: self.inner.module_id.clone(),
3756            })?;
3757        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3758            module_id: self.inner.module_id.clone(),
3759        })?
3760    }
3761
3762    pub async fn stop(&self) -> Result<(), SuperviseError> {
3763        self.stop_requested(None).await
3764    }
3765
3766    pub(crate) async fn stop_requested(
3767        &self,
3768        requester: Option<Principal>,
3769    ) -> Result<(), SuperviseError> {
3770        match self.state()? {
3771            ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3772            ModuleState::Starting
3773            | ModuleState::Running
3774            | ModuleState::Unresponsive
3775            | ModuleState::Restarting
3776            | ModuleState::Draining
3777            | ModuleState::Disabled => {}
3778        }
3779
3780        let (reply_tx, reply_rx) = oneshot::channel();
3781        self.inner
3782            .commands
3783            .send(SupervisorCommand::Drain {
3784                requester,
3785                reply: reply_tx,
3786            })
3787            .await
3788            .map_err(|_| SuperviseError::CommandClosed {
3789                module_id: self.inner.module_id.clone(),
3790            })?;
3791        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3792            module_id: self.inner.module_id.clone(),
3793        })?
3794    }
3795
3796    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3797        self.restart_requested(drain_timeout_ms, None).await
3798    }
3799
3800    pub(crate) async fn restart_requested(
3801        &self,
3802        drain_timeout_ms: Option<u64>,
3803        requester: Option<Principal>,
3804    ) -> Result<(), SuperviseError> {
3805        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3806        let (reply_tx, reply_rx) = oneshot::channel();
3807        self.inner
3808            .commands
3809            .send(SupervisorCommand::Restart {
3810                requester,
3811                drain_timeout_ms,
3812                received_at_generation,
3813                queued_at: Instant::now(),
3814                reply: reply_tx,
3815            })
3816            .await
3817            .map_err(|_| SuperviseError::CommandClosed {
3818                module_id: self.inner.module_id.clone(),
3819            })?;
3820        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3821            module_id: self.inner.module_id.clone(),
3822        })?
3823    }
3824
3825    /// Blue/green restart: see [`SupervisorCommand::Swap`] and the
3826    /// `supervisor_swap` module. Returns once the swap has cut over (the old
3827    /// process then drains in the background of the supervise loop) or has
3828    /// failed, leaving the old process serving.
3829    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3830        self.swap_requested(ready_timeout, None).await
3831    }
3832
3833    pub(crate) async fn swap_requested(
3834        &self,
3835        ready_timeout: Option<Duration>,
3836        requester: Option<Principal>,
3837    ) -> Result<(), SuperviseError> {
3838        let (reply_tx, reply_rx) = oneshot::channel();
3839        self.inner
3840            .commands
3841            .send(SupervisorCommand::Swap {
3842                requester,
3843                ready_timeout,
3844                reply: reply_tx,
3845            })
3846            .await
3847            .map_err(|_| SuperviseError::CommandClosed {
3848                module_id: self.inner.module_id.clone(),
3849            })?;
3850        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3851            module_id: self.inner.module_id.clone(),
3852        })?
3853    }
3854
3855    pub async fn reload(&self) -> Result<(), SuperviseError> {
3856        self.reload_requested(None).await
3857    }
3858
3859    pub(crate) async fn reload_requested(
3860        &self,
3861        requester: Option<Principal>,
3862    ) -> Result<(), SuperviseError> {
3863        let (reply_tx, reply_rx) = oneshot::channel();
3864        self.inner
3865            .commands
3866            .send(SupervisorCommand::Reload {
3867                requester,
3868                reply: reply_tx,
3869            })
3870            .await
3871            .map_err(|_| SuperviseError::CommandClosed {
3872                module_id: self.inner.module_id.clone(),
3873            })?;
3874        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3875            module_id: self.inner.module_id.clone(),
3876        })?
3877    }
3878
3879    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3880        self.set_enabled_requested(enabled, None).await
3881    }
3882
3883    pub(crate) async fn set_enabled_requested(
3884        &self,
3885        enabled: bool,
3886        requester: Option<Principal>,
3887    ) -> Result<bool, SuperviseError> {
3888        let (reply_tx, reply_rx) = oneshot::channel();
3889        self.inner
3890            .commands
3891            .send(SupervisorCommand::SetEnabled {
3892                requester,
3893                enabled,
3894                reply: reply_tx,
3895            })
3896            .await
3897            .map_err(|_| SuperviseError::CommandClosed {
3898                module_id: self.inner.module_id.clone(),
3899            })?;
3900        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3901            module_id: self.inner.module_id.clone(),
3902        })?
3903    }
3904
3905    /// The current process's protocol, or the configured protocol when down.
3906    /// A rescan stores the next launch spec without changing how an existing
3907    /// process registers, serves routes, is probed, or exits.
3908    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
3909        let configured = self
3910            .inner
3911            .configuration
3912            .lock()
3913            .map_err(|_| SuperviseError::StatePoisoned {
3914                module_id: Some(self.inner.module_id.clone()),
3915            })?
3916            .spec
3917            .protocol;
3918        let state = lock_snapshot(&self.inner.snapshot)?;
3919        Ok(state.spawned_protocol.unwrap_or(configured))
3920    }
3921
3922    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
3923        let configuration =
3924            self.inner
3925                .configuration
3926                .lock()
3927                .map_err(|_| SuperviseError::StatePoisoned {
3928                    module_id: Some(self.inner.module_id.clone()),
3929                })?;
3930        Ok((configuration.spec.clone(), configuration.health.clone()))
3931    }
3932
3933    /// Replace this module's launch spec, keeping its health and drain policy,
3934    /// the way a rescan does for a changed config entry. The running process is
3935    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
3936    #[cfg(any(test, feature = "test-support"))]
3937    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
3938        let (_, health) = self.configuration()?;
3939        let drain_timeout_ms = u64::try_from(
3940            self.inner
3941                .effective_drain_timeout
3942                .lock()
3943                .unwrap_or_else(|poisoned| poisoned.into_inner())
3944                .as_millis(),
3945        )
3946        .ok();
3947        self.update_configuration(spec, health, drain_timeout_ms)
3948            .await
3949    }
3950
3951    pub(crate) async fn update_configuration(
3952        &self,
3953        spec: ModuleSpec,
3954        health: HealthConfig,
3955        drain_timeout_ms: Option<u64>,
3956    ) -> Result<(), SuperviseError> {
3957        if spec.module_id != self.inner.module_id {
3958            return Err(SuperviseError::InvalidSpec {
3959                reason: "a supervised module's module_id cannot be changed".to_string(),
3960            });
3961        }
3962        validate_spec(&spec)?;
3963        let (reply_tx, reply_rx) = oneshot::channel();
3964        self.inner
3965            .commands
3966            .send(SupervisorCommand::UpdateConfiguration {
3967                spec: spec.clone(),
3968                health: health.clone(),
3969                drain_timeout_ms,
3970                reply: reply_tx,
3971            })
3972            .await
3973            .map_err(|_| SuperviseError::CommandClosed {
3974                module_id: self.inner.module_id.clone(),
3975            })?;
3976        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3977            module_id: self.inner.module_id.clone(),
3978        })?;
3979        let mut configuration =
3980            self.inner
3981                .configuration
3982                .lock()
3983                .map_err(|_| SuperviseError::StatePoisoned {
3984                    module_id: Some(self.inner.module_id.clone()),
3985                })?;
3986        configuration.spec = spec;
3987        configuration.health = health;
3988        Ok(())
3989    }
3990}
3991
3992impl Drop for SupervisedModuleInner {
3993    fn drop(&mut self) {
3994        let Ok(mut monitor) = self.monitor.lock() else {
3995            return;
3996        };
3997        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
3998            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
3999                state.state = ModuleState::Stopped;
4000                clear_current_process_facts(state);
4001            });
4002            monitor.abort();
4003        }
4004        let _ = monitor.take();
4005    }
4006}
4007
4008#[derive(Debug)]
4009enum SupervisorCommand {
4010    Drain {
4011        requester: Option<Principal>,
4012        reply: oneshot::Sender<Result<(), SuperviseError>>,
4013    },
4014    Retire {
4015        requester: Option<Principal>,
4016        reply: oneshot::Sender<Result<(), SuperviseError>>,
4017    },
4018    Restart {
4019        requester: Option<Principal>,
4020        /// Operator override for this one restart's drain budget, in ms. `None`
4021        /// uses the module's configured/default budget; `Some(0)` cuts
4022        /// immediately (wedge bounce: a stuck request never settles, so
4023        /// waiting only delays recovery).
4024        drain_timeout_ms: Option<u64>,
4025        /// The module's `spawn_generation` when the request was received, before
4026        /// it waited in the command queue. A queued restart whose module has
4027        /// since spawned a newer process is already satisfied (see the handler).
4028        received_at_generation: u64,
4029        /// When the request entered the command queue, so the handler can log
4030        /// how long it waited behind the loop's other work.
4031        queued_at: Instant,
4032        reply: oneshot::Sender<Result<(), SuperviseError>>,
4033    },
4034    Reload {
4035        requester: Option<Principal>,
4036        reply: oneshot::Sender<Result<(), SuperviseError>>,
4037    },
4038    SetEnabled {
4039        requester: Option<Principal>,
4040        enabled: bool,
4041        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4042    },
4043    UpdateConfiguration {
4044        spec: ModuleSpec,
4045        health: HealthConfig,
4046        /// Per-module drain override from the new config; `None` re-resolves to
4047        /// the supervisor-wide default.
4048        drain_timeout_ms: Option<u64>,
4049        reply: oneshot::Sender<()>,
4050    },
4051    Swap {
4052        requester: Option<Principal>,
4053        /// How long the candidate may take to register and declare itself
4054        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4055        ready_timeout: Option<Duration>,
4056        /// Answered at cutover or failure; the incumbent's drain follows.
4057        reply: oneshot::Sender<Result<(), SuperviseError>>,
4058    },
4059}
4060
4061#[derive(Debug)]
4062pub enum SuperviseError {
4063    InvalidSpec {
4064        reason: String,
4065    },
4066    Spawn {
4067        program: PathBuf,
4068        source: io::Error,
4069        cgroup_path: Option<PathBuf>,
4070    },
4071    Cgroup {
4072        module_id: String,
4073        source: io::Error,
4074    },
4075    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4076    /// than spawn a reserved module without its identity binding.
4077    LaunchNonce {
4078        reason: String,
4079    },
4080    Wait {
4081        module_id: String,
4082        source: io::Error,
4083    },
4084    Kill {
4085        module_id: String,
4086        source: io::Error,
4087    },
4088    Forwarding(ForwardingError),
4089    Registry(RegistryError),
4090    ReloadUnavailable {
4091        module_id: String,
4092        reason: String,
4093    },
4094    /// An operator restart/reload was requested for a module that is currently
4095    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4096    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4097    /// by a restart, so these commands are rejected instead of re-enabling it.
4098    Disabled {
4099        module_id: String,
4100    },
4101    ReloadFailed {
4102        module_id: String,
4103        reason: String,
4104    },
4105    RegistrationStillActive {
4106        module_id: String,
4107        waited: Duration,
4108    },
4109    StatePoisoned {
4110        module_id: Option<String>,
4111    },
4112    CommandClosed {
4113        module_id: String,
4114    },
4115    /// A restart or reload arrived while a swap's candidate was warming. The
4116    /// swap owns the module until it cuts over or fails; a stop or disable
4117    /// would have aborted it instead.
4118    SwapInProgress {
4119        module_id: String,
4120    },
4121    /// A swap was refused before anything was spawned.
4122    SwapRefused {
4123        module_id: String,
4124        reason: SwapRefusal,
4125    },
4126    /// A swap spawned a candidate and gave up on it. The candidate has been
4127    /// killed and its slot freed; the incumbent was left serving and was never
4128    /// drained, except in the one `CutoverLost` case described on that arm.
4129    SwapFailed {
4130        module_id: String,
4131        arm: SwapFailureArm,
4132        detail: String,
4133        /// How the candidate exited, when it exited on its own before the
4134        /// supervisor gave up on it.
4135        candidate_exit: Option<ExitReport>,
4136    },
4137}
4138
4139/// Why a swap was refused before a candidate was spawned.
4140#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4141pub enum SwapRefusal {
4142    /// The module's config does not declare `overlap: "safe"`.
4143    OverlapExclusive,
4144    /// The module is not registered, so there is no incumbent to keep serving
4145    /// and nothing a swap would improve on; a plain restart is the tool.
4146    NotRegistered,
4147    /// The module does not speak the subc wire, so a candidate could never
4148    /// register or declare itself ready.
4149    ProtocolNone,
4150    /// The supervisor lacks the forwarding table (to cut routes over) or the
4151    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4152    NotConfigured,
4153    /// A swap is already open for this module.
4154    AlreadySwapping,
4155}
4156
4157impl SwapRefusal {
4158    pub fn as_str(self) -> &'static str {
4159        match self {
4160            Self::OverlapExclusive => "overlap_exclusive",
4161            Self::NotRegistered => "not_registered",
4162            Self::ProtocolNone => "protocol_none",
4163            Self::NotConfigured => "not_configured",
4164            Self::AlreadySwapping => "already_swapping",
4165        }
4166    }
4167}
4168
4169/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4170/// serving and undrained; see `CutoverLost`.
4171#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4172pub enum SwapFailureArm {
4173    /// The candidate process could not be started.
4174    SpawnFailed,
4175    /// The candidate did not register within the readiness budget.
4176    NeverRegistered,
4177    /// The candidate registered but did not declare itself ready in time.
4178    NeverReady,
4179    /// The candidate exited before cutover.
4180    CandidateExited,
4181    /// The candidate declared itself ready but failed its health probe.
4182    CandidateUnhealthy,
4183    /// An operator stop, disable or retire arrived while the candidate warmed.
4184    /// The candidate was killed and the operator's command then carried out on
4185    /// the incumbent.
4186    Interrupted,
4187    /// The candidate's connection closed at the moment of cutover. If it
4188    /// closed before forwarding moved, the incumbent is untouched. If it closed
4189    /// between the forwarding and registry halves of cutover, forwarding can no
4190    /// longer route to the incumbent, so the module is restarted plainly.
4191    CutoverLost,
4192}
4193
4194impl SwapFailureArm {
4195    pub fn as_str(self) -> &'static str {
4196        match self {
4197            Self::SpawnFailed => "spawn_failed",
4198            Self::NeverRegistered => "never_registered",
4199            Self::NeverReady => "never_ready",
4200            Self::CandidateExited => "candidate_exited",
4201            Self::CandidateUnhealthy => "candidate_unhealthy",
4202            Self::Interrupted => "interrupted",
4203            Self::CutoverLost => "cutover_lost",
4204        }
4205    }
4206}
4207
4208impl fmt::Display for SuperviseError {
4209    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4210        match self {
4211            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4212            Self::Spawn {
4213                program,
4214                source,
4215                cgroup_path: Some(cgroup_path),
4216            } => write!(
4217                f,
4218                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4219                cgroup_path.display(),
4220                program.display()
4221            ),
4222            Self::Spawn {
4223                program,
4224                source,
4225                cgroup_path: None,
4226            } => write!(
4227                f,
4228                "failed to spawn module '{}': {source}",
4229                program.display()
4230            ),
4231            Self::Cgroup { module_id, source } => {
4232                write!(
4233                    f,
4234                    "failed to prepare cgroup for module '{module_id}': {source}"
4235                )
4236            }
4237            Self::LaunchNonce { reason } => {
4238                write!(
4239                    f,
4240                    "failed to generate reserved-module launch nonce: {reason}"
4241                )
4242            }
4243            Self::Wait { module_id, source } => {
4244                write!(f, "failed to wait for module '{module_id}': {source}")
4245            }
4246            Self::Kill { module_id, source } => {
4247                write!(f, "failed to kill module '{module_id}': {source}")
4248            }
4249            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4250            Self::Registry(err) => write!(f, "registry error: {err}"),
4251            Self::ReloadUnavailable { module_id, reason } => {
4252                write!(f, "reload unavailable for module '{module_id}': {reason}")
4253            }
4254            Self::Disabled { module_id } => {
4255                write!(
4256                    f,
4257                    "module '{module_id}' is disabled; enable it before restart or reload"
4258                )
4259            }
4260            Self::ReloadFailed { module_id, reason } => {
4261                write!(f, "reload failed for module '{module_id}': {reason}")
4262            }
4263            Self::RegistrationStillActive { module_id, waited } => write!(
4264                f,
4265                "module '{module_id}' registration remained active after waiting {waited:?}"
4266            ),
4267            Self::StatePoisoned { module_id } => match module_id {
4268                Some(module_id) => {
4269                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4270                }
4271                None => write!(f, "supervisor state was poisoned"),
4272            },
4273            Self::CommandClosed { module_id } => {
4274                write!(
4275                    f,
4276                    "supervisor command channel for module '{module_id}' is closed"
4277                )
4278            }
4279            Self::SwapInProgress { module_id } => write!(
4280                f,
4281                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4282            ),
4283            Self::SwapRefused { module_id, reason } => match reason {
4284                SwapRefusal::OverlapExclusive => write!(
4285                    f,
4286                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4287                ),
4288                SwapRefusal::NotRegistered => write!(
4289                    f,
4290                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4291                ),
4292                SwapRefusal::ProtocolNone => write!(
4293                    f,
4294                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4295                ),
4296                SwapRefusal::NotConfigured => write!(
4297                    f,
4298                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4299                ),
4300                SwapRefusal::AlreadySwapping => {
4301                    write!(f, "module '{module_id}' is already being swapped")
4302                }
4303            },
4304            Self::SwapFailed {
4305                module_id,
4306                arm,
4307                detail,
4308                ..
4309            } => write!(
4310                f,
4311                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4312                arm.as_str()
4313            ),
4314        }
4315    }
4316}
4317
4318impl Error for SuperviseError {
4319    fn source(&self) -> Option<&(dyn Error + 'static)> {
4320        match self {
4321            Self::Spawn { source, .. }
4322            | Self::Cgroup { source, .. }
4323            | Self::Wait { source, .. }
4324            | Self::Kill { source, .. } => Some(source),
4325            Self::Forwarding(err) => Some(err),
4326            Self::Registry(err) => Some(err),
4327            Self::LaunchNonce { .. }
4328            | Self::InvalidSpec { .. }
4329            | Self::ReloadUnavailable { .. }
4330            | Self::Disabled { .. }
4331            | Self::ReloadFailed { .. }
4332            | Self::RegistrationStillActive { .. }
4333            | Self::StatePoisoned { .. }
4334            | Self::CommandClosed { .. }
4335            | Self::SwapInProgress { .. }
4336            | Self::SwapRefused { .. }
4337            | Self::SwapFailed { .. } => None,
4338        }
4339    }
4340}
4341
4342pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4343    if spec.module_id.trim().is_empty() {
4344        return Err(SuperviseError::InvalidSpec {
4345            reason: "module_id must not be empty".to_string(),
4346        });
4347    }
4348
4349    Ok(())
4350}
4351
4352#[derive(Debug, Default)]
4353struct HealthProbeRuntime {
4354    configured_health: Option<HealthConfig>,
4355    registered_connection: Option<crate::ConnectionId>,
4356    advertised: bool,
4357    next_probe_at: Option<Instant>,
4358    probe_index: u64,
4359}
4360
4361fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4362    lock_snapshot(snapshot)
4363        .ok()
4364        .and_then(|state| state.spawned_protocol)
4365        .unwrap_or(spec.protocol)
4366}
4367
4368impl HealthProbeRuntime {
4369    fn refresh_registration(
4370        &mut self,
4371        spec: &ModuleSpec,
4372        runtime: &SupervisorRuntimeConfig,
4373        registry: &Registry,
4374        snapshot: &SharedSnapshot,
4375    ) {
4376        if self.configured_health.as_ref() != Some(&runtime.health) {
4377            self.configured_health = Some(runtime.health.clone());
4378            self.next_probe_at = None;
4379            self.registered_connection = None;
4380            self.probe_index = 0;
4381        }
4382        // A non-wire process never registers. Only an explicitly configured
4383        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4384        // health failure for that kind of process.
4385        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4386            self.registered_connection = None;
4387            self.advertised = runtime.health.http.is_some();
4388            if !self.advertised {
4389                self.next_probe_at = None;
4390                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4391                    let unknown = ModuleHealthStatus::default();
4392                    if state.health != unknown {
4393                        state.health = unknown;
4394                    }
4395                });
4396            } else if self.next_probe_at.is_none() {
4397                self.next_probe_at = Some(
4398                    Instant::now()
4399                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4400                );
4401            }
4402            return;
4403        }
4404
4405        let registration = match registry.get_module(&spec.module_id) {
4406            Ok(registration) => registration,
4407            Err(err) => {
4408                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4409                self.advertised = false;
4410                self.next_probe_at = None;
4411                return;
4412            }
4413        };
4414
4415        let Some(registration) = registration else {
4416            self.registered_connection = None;
4417            self.advertised = false;
4418            self.next_probe_at = None;
4419            return;
4420        };
4421
4422        let advertised = registration
4423            .control_ops
4424            .iter()
4425            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4426        if !advertised {
4427            self.registered_connection = Some(registration.connection_id);
4428            self.advertised = false;
4429            self.next_probe_at = None;
4430            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4431                if state.health.status != SupervisorHealthStatus::Unknown
4432                    || state.health.consecutive_failures != 0
4433                    || state.health.last_probe_ms.is_some()
4434                    || state.health.detail.is_some()
4435                    || state.health.metrics.is_some()
4436                {
4437                    state.health.status = SupervisorHealthStatus::Unknown;
4438                    state.health.consecutive_failures = 0;
4439                    state.health.last_probe_ms = None;
4440                    state.health.detail = None;
4441                    state.health.metrics = None;
4442                }
4443            });
4444            return;
4445        }
4446
4447        let reregistered = self.registered_connection != Some(registration.connection_id);
4448        self.registered_connection = Some(registration.connection_id);
4449        self.advertised = true;
4450        if reregistered || self.next_probe_at.is_none() {
4451            self.probe_index = 0;
4452            self.next_probe_at = Some(
4453                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4454            );
4455            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4456                state.health.status = SupervisorHealthStatus::Unknown;
4457                state.health.consecutive_failures = 0;
4458                state.health.detail = None;
4459                state.health.metrics = None;
4460            });
4461        }
4462    }
4463
4464    fn wake_after(&self) -> Option<Duration> {
4465        if !self.advertised {
4466            return None;
4467        }
4468        self.next_probe_at
4469            .map(|next| next.saturating_duration_since(Instant::now()))
4470    }
4471
4472    fn due(&self) -> bool {
4473        self.advertised
4474            && self
4475                .next_probe_at
4476                .is_some_and(|next| Instant::now() >= next)
4477    }
4478
4479    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4480        self.probe_index = self.probe_index.wrapping_add(1);
4481        self.next_probe_at = Some(
4482            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4483        );
4484    }
4485}
4486
4487/// What a failed health probe actually OBSERVED, kept apart from how it reads.
4488///
4489/// This was a struct with a single `message: String`, and every one of the
4490/// fifteen construction sites collapsed into it. Each site knows exactly what it
4491/// saw -- the lane is gone, the module did not answer in time, the module
4492/// answered with the wrong thing -- and `handle_health_probe_failure` then
4493/// treated all of them identically: increment a counter, compare to a threshold,
4494/// restart the module. THE DISTINCTION EXISTED AT EVERY CALL SITE AND WAS
4495/// DESTROYED BEFORE THE DECISION THAT NEEDED IT.
4496///
4497/// The distinction that matters is not severity, it is EVIDENTIAL WEIGHT:
4498///
4499/// * `LaneDead` is PROOF. The module's control connection is gone; nothing will
4500///   answer on it again.
4501/// * `NoAnswer` is ABSENCE OF EVIDENCE. It is consistent with a wedged module
4502///   AND with a perfectly healthy one that lost a CPU race -- which is what
4503///   happens under machine load, and is how this supervisor killed a healthy
4504///   module three times in one day.
4505/// * `BadAnswer` proves the module is ALIVE. It replied; the reply was wrong.
4506///   Restarting on it is defensible, but it is not the silence case and should
4507///   never be counted as one.
4508/// * `Misconfigured` is a daemon-side fault. The module has not been asked
4509///   anything, so it cannot be evidence about the module at all.
4510///
4511/// The asymmetry is the whole point: under saturation the WEAKEST signal is the
4512/// one that fires most often, and while every variant collapsed into one string
4513/// it carried the same weight as the strongest.
4514///
4515/// LIVE BEHAVIOUR TODAY, stated here because this doc block describes the
4516/// DESIGN and a reader stopping at it gets the build backwards: the restart
4517/// decision does NOT yet consult this classification -- consecutive `NoAnswer`
4518/// probes still increment the failure streak and drive escalation at the
4519/// threshold (see `is_proof_of_death` below for why that is deliberate and
4520/// what gates the change). Absence of evidence restarts modules today.
4521#[derive(Debug)]
4522enum HealthProbeEvidence {
4523    /// The module's control lane is gone. Proof of death.
4524    LaneDead,
4525    /// No reply within the deadline. Proves nothing about the module's state.
4526    NoAnswer,
4527    /// The module replied, but not with a usable health report. Proves it is alive.
4528    BadAnswer,
4529    /// The daemon could not ask. Says nothing about the module.
4530    Misconfigured,
4531}
4532
4533#[derive(Debug)]
4534struct HealthProbeError {
4535    evidence: HealthProbeEvidence,
4536    message: String,
4537}
4538
4539impl HealthProbeError {
4540    fn lane_dead(message: impl Into<String>) -> Self {
4541        Self::with(HealthProbeEvidence::LaneDead, message)
4542    }
4543
4544    fn no_answer(message: impl Into<String>) -> Self {
4545        Self::with(HealthProbeEvidence::NoAnswer, message)
4546    }
4547
4548    fn bad_answer(message: impl Into<String>) -> Self {
4549        Self::with(HealthProbeEvidence::BadAnswer, message)
4550    }
4551
4552    fn misconfigured(message: impl Into<String>) -> Self {
4553        Self::with(HealthProbeEvidence::Misconfigured, message)
4554    }
4555
4556    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4557        Self {
4558            evidence,
4559            message: message.into(),
4560        }
4561    }
4562
4563    /// Whether this observation is proof the module cannot serve.
4564    ///
4565    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4566    /// variant that fires under CPU starvation, and treating it as proof is the
4567    /// defect this enum exists to make impossible to reintroduce silently.
4568    ///
4569    /// NOT YET CONSULTED BY THE RESTART DECISION, deliberately. Requiring proof
4570    /// to restart also needs a bound for the case it excludes -- a genuinely
4571    /// wedged module, alive but never answering -- and that bound must come from
4572    /// the distribution of real late-answer latencies, which nothing measures
4573    /// yet. Landing the classification first makes the later change a one-line
4574    /// decision against evidence that already exists, rather than two unproven
4575    /// changes at once.
4576    #[allow(dead_code)]
4577    fn is_proof_of_death(&self) -> bool {
4578        matches!(self.evidence, HealthProbeEvidence::LaneDead)
4579    }
4580
4581    /// Short stable label for logs and the health snapshot.
4582    ///
4583    /// An operator reading `ck health` currently cannot tell "the module is gone"
4584    /// from "the module did not answer in five seconds", because both render as
4585    /// prose in the same field. These labels are what make the two
4586    /// distinguishable at a glance, and they are what a later restart-policy
4587    /// change will be argued from.
4588    fn label(&self) -> &'static str {
4589        match self.evidence {
4590            HealthProbeEvidence::LaneDead => "lane-dead",
4591            HealthProbeEvidence::NoAnswer => "no-answer",
4592            HealthProbeEvidence::BadAnswer => "bad-answer",
4593            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4594        }
4595    }
4596}
4597
4598impl fmt::Display for HealthProbeError {
4599    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4600        f.write_str(&self.message)
4601    }
4602}
4603
4604async fn run_health_probe_cycle(
4605    spec: &ModuleSpec,
4606    runtime: &SupervisorRuntimeConfig,
4607    registry: &Registry,
4608    process_liveness: &SupervisorProcessLiveness,
4609    snapshot: &SharedSnapshot,
4610    child: &mut Option<SupervisedChild>,
4611) {
4612    let now_ms = unix_ms_now();
4613    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4614        .then_some(runtime.health.http.as_deref())
4615        .flatten();
4616    let result = match http {
4617        Some(url) => probe_http_health(url, runtime.health.deadline).await,
4618        None => probe_module_health(&spec.module_id, runtime, None).await,
4619    };
4620    match result {
4621        Ok(report) => {
4622            handle_health_report(
4623                spec,
4624                runtime,
4625                registry,
4626                process_liveness,
4627                snapshot,
4628                child,
4629                report,
4630                now_ms,
4631            )
4632            .await;
4633        }
4634        Err(err) => {
4635            if http.is_some() {
4636                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4637                    state.health.status = SupervisorHealthStatus::Failing;
4638                });
4639            }
4640            handle_health_probe_failure(
4641                spec,
4642                runtime,
4643                registry,
4644                process_liveness,
4645                snapshot,
4646                child,
4647                err,
4648                now_ms,
4649            )
4650            .await;
4651        }
4652    }
4653}
4654
4655pub(crate) struct HttpProbeTarget<'a> {
4656    address: std::net::SocketAddr,
4657    localhost: bool,
4658    authority: &'a str,
4659    path: String,
4660}
4661
4662/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
4663/// or TLS. A URL cannot turn a local health check into an outbound connection.
4664pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4665    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4666        return Err("must not contain whitespace, controls, or a fragment".into());
4667    }
4668    let rest = url
4669        .strip_prefix("http://")
4670        .ok_or("must use plain http://")?;
4671    let split = rest.find(['/', '?']).unwrap_or(rest.len());
4672    let (authority, suffix) = rest.split_at(split);
4673    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4674        ("::1", rest)
4675    } else {
4676        let split = authority.find(':').unwrap_or(authority.len());
4677        authority.split_at(split)
4678    };
4679    let ip: std::net::IpAddr = match host {
4680        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4681        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4682        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4683    };
4684    let port = if port.is_empty() {
4685        80
4686    } else {
4687        port.strip_prefix(':')
4688            .and_then(|p| p.parse::<u16>().ok())
4689            .filter(|p| *p > 0)
4690            .ok_or("must have a valid nonzero TCP port")?
4691    };
4692    let path = if suffix.is_empty() {
4693        "/".into()
4694    } else if suffix.starts_with('?') {
4695        format!("/{suffix}")
4696    } else {
4697        suffix.into()
4698    };
4699    Ok(HttpProbeTarget {
4700        address: std::net::SocketAddr::new(ip, port),
4701        localhost: host == "localhost",
4702        authority,
4703        path,
4704    })
4705}
4706
4707async fn probe_http_health(
4708    url: &str,
4709    deadline: Duration,
4710) -> Result<HealthReport, HealthProbeError> {
4711    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4712    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4713    // Keep partial diagnostics outside the timed future so cancellation does
4714    // not discard a status line or body bytes already received.
4715    let mut response_status = String::new();
4716    let mut body = Vec::new();
4717    let probe = async {
4718        // Resolve localhost ourselves so a hosts-file override cannot turn
4719        // this into an outbound request, while IPv6-only local servers work.
4720        let connection = match tokio::net::TcpStream::connect(target.address).await {
4721            Err(_) if target.localhost => {
4722                tokio::net::TcpStream::connect((
4723                    std::net::Ipv6Addr::LOCALHOST,
4724                    target.address.port(),
4725                ))
4726                .await
4727            }
4728            result => result,
4729        };
4730        let mut stream = connection.map_err(|error| {
4731            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4732        })?;
4733        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4734            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4735        let mut reader = BufReader::new(stream);
4736        let mut budget = 16 * 1024;
4737        let status = http_line(&mut reader, &mut budget).await?;
4738        let mut words = status.split_ascii_whitespace();
4739        let version = words.next();
4740        let code = words
4741            .next()
4742            .filter(|word| word.len() == 3)
4743            .and_then(|word| word.parse::<u16>().ok());
4744        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4745            || !code.is_some_and(|code| (100..600).contains(&code))
4746        {
4747            return Err(HealthProbeError::bad_answer(format!(
4748                "invalid HTTP status: {status}"
4749            )));
4750        }
4751        let code = code.expect("validated status code");
4752        response_status = status.clone();
4753        let mut length = None;
4754        let mut chunked = false;
4755        loop {
4756            let line = http_line(&mut reader, &mut budget).await?;
4757            if line.is_empty() {
4758                break;
4759            }
4760            if let Some((name, value)) = line.split_once(':') {
4761                if name.eq_ignore_ascii_case("content-length") {
4762                    length = Some(value.trim().parse::<u64>().map_err(|_| {
4763                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
4764                    })?);
4765                } else if name.eq_ignore_ascii_case("transfer-encoding") {
4766                    chunked = value.trim().eq_ignore_ascii_case("chunked");
4767                }
4768            }
4769        }
4770        if chunked {
4771            while body.len() < 200 {
4772                let line = http_line(&mut reader, &mut budget).await?;
4773                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4774                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4775                if size == 0 {
4776                    break;
4777                }
4778                let count = size.min((200 - body.len()) as u64) as usize;
4779                let start = body.len();
4780                (&mut reader)
4781                    .take(count as u64)
4782                    .read_to_end(&mut body)
4783                    .await
4784                    .map_err(|error| {
4785                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4786                    })?;
4787                if body.len() - start != count {
4788                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4789                }
4790                if size > count as u64 || body.len() == 200 {
4791                    break;
4792                }
4793                if !http_line(&mut reader, &mut budget).await?.is_empty() {
4794                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4795                }
4796            }
4797        } else {
4798            reader
4799                .take(length.unwrap_or(200).min(200))
4800                .read_to_end(&mut body)
4801                .await
4802                .map_err(|error| {
4803                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4804                })?;
4805        }
4806        if (200..300).contains(&code) {
4807            Ok(HealthReport::ok())
4808        } else {
4809            Err(HealthProbeError::bad_answer(
4810                "HTTP health endpoint returned non-2xx",
4811            ))
4812        }
4813    };
4814    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4815        Err(HealthProbeError::no_answer(format!(
4816            "HTTP probe timed out after {deadline:?}"
4817        )))
4818    });
4819    if let Err(error) = &mut result {
4820        if !response_status.is_empty() {
4821            error.message = format!(
4822                "{}; {response_status}: {}",
4823                error.message,
4824                String::from_utf8_lossy(&body)
4825            );
4826        }
4827    }
4828    result
4829}
4830
4831async fn http_line(
4832    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4833    remaining: &mut usize,
4834) -> Result<String, HealthProbeError> {
4835    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4836    let mut line = Vec::new();
4837    (&mut *reader)
4838        .take(*remaining as u64)
4839        .read_until(b'\n', &mut line)
4840        .await
4841        .map_err(|error| {
4842            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4843        })?;
4844    *remaining -= line.len();
4845    if !line.ends_with(b"\r\n") {
4846        return Err(HealthProbeError::bad_answer(
4847            "HTTP headers are incomplete or exceed 16 KiB",
4848        ));
4849    }
4850    line.truncate(line.len() - 2);
4851    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4852}
4853
4854async fn probe_module_health(
4855    module_id: &str,
4856    runtime: &SupervisorRuntimeConfig,
4857    drain_deadline: Option<Instant>,
4858) -> Result<HealthReport, HealthProbeError> {
4859    let Some(forwarding) = runtime.forwarding.as_ref() else {
4860        return Err(HealthProbeError::misconfigured(
4861            "supervisor was not configured with a forwarding table",
4862        ));
4863    };
4864    let probe_started_at = Instant::now();
4865    let mut deadline = probe_started_at + runtime.health.deadline;
4866    if let Some(drain_deadline) = drain_deadline {
4867        deadline = deadline.min(drain_deadline);
4868    }
4869    let pending = if drain_deadline.is_some() {
4870        forwarding.begin_drain_health_probe_rpc_for(
4871            module_id,
4872            MODULE_CONTROL_OP_HEALTH_CHECK,
4873            probe_started_at,
4874            deadline,
4875        )
4876    } else {
4877        forwarding.begin_health_probe_rpc_for(
4878            module_id,
4879            MODULE_CONTROL_OP_HEALTH_CHECK,
4880            probe_started_at,
4881            deadline,
4882        )
4883    }
4884    .map_err(|err| {
4885        // The endpoint is not registered, so there is no live control lane to
4886        // ask. That is the module being absent, not slow.
4887        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4888    })?;
4889    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4890}
4891
4892/// [`probe_module_health`] for one endpoint rather than the id's active one.
4893///
4894/// A swap probes two processes that no by-id lookup reaches: its candidate
4895/// before cutover, and its superseded incumbent (for busy gauges) while the
4896/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
4897/// bounds the by-id drain probe.
4898async fn probe_endpoint_health(
4899    endpoint: crate::ModuleEndpointId,
4900    runtime: &SupervisorRuntimeConfig,
4901    deadline_cap: Option<Instant>,
4902) -> Result<HealthReport, HealthProbeError> {
4903    let Some(forwarding) = runtime.forwarding.as_ref() else {
4904        return Err(HealthProbeError::misconfigured(
4905            "supervisor was not configured with a forwarding table",
4906        ));
4907    };
4908    let probe_started_at = Instant::now();
4909    let mut deadline = probe_started_at + runtime.health.deadline;
4910    if let Some(cap) = deadline_cap {
4911        deadline = deadline.min(cap);
4912    }
4913    let pending = forwarding
4914        .begin_endpoint_health_probe_rpc_for(
4915            endpoint,
4916            MODULE_CONTROL_OP_HEALTH_CHECK,
4917            probe_started_at,
4918            deadline,
4919        )
4920        .map_err(|err| {
4921            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4922        })?;
4923    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4924}
4925
4926/// Send a begun health probe and classify its answer.
4927async fn await_health_probe(
4928    forwarding: &ForwardingTable,
4929    pending: PendingModuleControlRpc,
4930    deadline: Instant,
4931    probe_budget: Duration,
4932) -> Result<HealthReport, HealthProbeError> {
4933    let PendingModuleControlRpc {
4934        endpoint,
4935        module_sink,
4936        negotiated_ver,
4937        corr,
4938        receiver,
4939    } = pending;
4940    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4941        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
4942    })?;
4943    let frame = Frame::build_with_version(
4944        negotiated_ver,
4945        FrameType::Request,
4946        control_flags(),
4947        0,
4948        0,
4949        corr,
4950        body,
4951    )
4952    .map_err(|err| {
4953        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
4954    })?;
4955
4956    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
4957    // blocks waiting for capacity when the module's egress queue is full, and an
4958    // unbounded await here freezes the whole supervision actor (it stops polling
4959    // Child::wait and supervisor commands), making the module unrecoverable
4960    // in-band. On timeout the probe fails like any transport failure.
4961    match timeout_at(deadline, module_sink.send(frame)).await {
4962        Ok(Ok(())) => {}
4963        Ok(Err(err)) => {
4964            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
4965            // A closed sink means the module's egress channel is gone -- the
4966            // receiving half is dropped when its connection tears down. Proof.
4967            return Err(HealthProbeError::lane_dead(format!(
4968                "failed to send health.check: {err}"
4969            )));
4970        }
4971        Err(_elapsed) => {
4972            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
4973            // A full egress queue means the module is not draining its socket, which
4974            // is consistent with a wedged module AND with one whose reader is merely
4975            // starved. Silence, not proof.
4976            return Err(HealthProbeError::no_answer(
4977                "health.check send timed out before enqueue (module egress full)",
4978            ));
4979        }
4980    }
4981
4982    match timeout_at(deadline, receiver).await {
4983        // Each arm records WHAT WAS OBSERVED. Four of them are the module
4984        // demonstrably answering -- rejected, non-health, malformed, wrong op --
4985        // and those prove it is alive even though the probe failed.
4986        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4987            response.health_report().ok_or_else(|| {
4988                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
4989            })
4990        }
4991        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
4992            format!("health.check rejected: {}", body.message),
4993        )),
4994        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4995            Err(HealthProbeError::lane_dead(message))
4996        }
4997        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4998            Err(HealthProbeError::bad_answer(message))
4999        }
5000        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5001            Err(HealthProbeError::bad_answer(format!(
5002                "expected module-control op '{expected}', got '{actual}'"
5003            )))
5004        }
5005        // A reply that crosses the deadline before this waiter observes it is
5006        // still proof of life. The forwarding path records its end-to-end latency
5007        // before delivering this classification.
5008        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5009            "module answered health.check after its daemon deadline",
5010        )),
5011        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5012            "health.check waiter was canceled before the module responded",
5013        )),
5014        Err(_) => {
5015            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5016            Err(HealthProbeError::no_answer(format!(
5017                "module did not answer health.check within {probe_budget:?}"
5018            )))
5019        }
5020    }
5021}
5022
5023#[allow(clippy::too_many_arguments)]
5024async fn handle_health_report(
5025    spec: &ModuleSpec,
5026    runtime: &SupervisorRuntimeConfig,
5027    registry: &Registry,
5028    process_liveness: &SupervisorProcessLiveness,
5029    snapshot: &SharedSnapshot,
5030    child: &mut Option<SupervisedChild>,
5031    report: HealthReport,
5032    now_ms: u64,
5033) {
5034    let status = supervisor_health_status(report.status);
5035    let detail = report.detail.clone();
5036    let metrics = truncate_health_metrics(report.metrics);
5037    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5038        state.health.status = status;
5039        state.health.last_probe_ms = Some(now_ms);
5040        state.health.detail = detail.clone();
5041        state.health.metrics = metrics.clone();
5042        state.health.consecutive_failures = 0;
5043    });
5044
5045    let action = match report.status {
5046        HealthStatus::Ok => return,
5047        HealthStatus::Degraded => runtime.health.on_degraded,
5048        HealthStatus::Failing => runtime.health.on_failing,
5049    };
5050    apply_l3_health_action(
5051        spec,
5052        runtime,
5053        registry,
5054        process_liveness,
5055        snapshot,
5056        child,
5057        status,
5058        detail.as_deref(),
5059        action,
5060        now_ms,
5061    )
5062    .await;
5063}
5064
5065#[allow(clippy::too_many_arguments)]
5066async fn handle_health_probe_failure(
5067    spec: &ModuleSpec,
5068    runtime: &SupervisorRuntimeConfig,
5069    registry: &Registry,
5070    process_liveness: &SupervisorProcessLiveness,
5071    snapshot: &SharedSnapshot,
5072    child: &mut Option<SupervisedChild>,
5073    err: HealthProbeError,
5074    now_ms: u64,
5075) {
5076    let threshold = runtime.health.failure_threshold.max(1);
5077    let mut failures = 0;
5078    // Carry the evidence class into the operator-visible detail. Without it,
5079    // "module did not answer within 5s" and "the control lane is gone" are two
5080    // prose strings in the same field, and the reader has to know the codebase to
5081    // tell which one is proof of anything.
5082    let detail = format!("[{}] {err}", err.label());
5083    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5084        // A failed wire probe invalidates the last report, even before the
5085        // restart threshold. HTTP probes already mark failures as Failing.
5086        if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5087            state.health.status = SupervisorHealthStatus::Unknown;
5088        }
5089        state.health.last_probe_ms = Some(now_ms);
5090        state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5091        state.health.detail = Some(detail.clone());
5092        state.health.metrics = None;
5093        failures = state.health.consecutive_failures;
5094    });
5095
5096    if failures < threshold {
5097        warn!(
5098            module_id = %spec.module_id,
5099            consecutive_failures = failures,
5100            threshold,
5101            evidence = err.label(),
5102            detail = %detail,
5103            "health.check probe failed"
5104        );
5105        return;
5106    }
5107
5108    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5109        state.state = ModuleState::Unresponsive;
5110        state.health.status = SupervisorHealthStatus::Unresponsive;
5111    });
5112    // The evidence class is logged at the kill site because this is the line an
5113    // operator reads after an unexplained restart. A streak of `no-answer` under
5114    // machine load is the known false-positive shape; a `lane-dead` is not.
5115    if runtime.health.critical {
5116        error!(
5117            module_id = %spec.module_id,
5118            status = "unresponsive",
5119            evidence = err.label(),
5120            detail = %detail,
5121            "critical module health alert"
5122        );
5123    } else {
5124        warn!(
5125            module_id = %spec.module_id,
5126            status = "unresponsive",
5127            evidence = err.label(),
5128            detail = %detail,
5129            "module health threshold breached"
5130        );
5131    }
5132    if let Err(err) = health_restart_child(
5133        spec,
5134        runtime,
5135        registry,
5136        process_liveness,
5137        snapshot,
5138        child,
5139        SupervisorHealthStatus::Unresponsive,
5140        Some(&detail),
5141        now_ms,
5142    )
5143    .await
5144    {
5145        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5146    }
5147}
5148
5149#[allow(clippy::too_many_arguments)]
5150async fn apply_l3_health_action(
5151    spec: &ModuleSpec,
5152    runtime: &SupervisorRuntimeConfig,
5153    registry: &Registry,
5154    process_liveness: &SupervisorProcessLiveness,
5155    snapshot: &SharedSnapshot,
5156    child: &mut Option<SupervisedChild>,
5157    status: SupervisorHealthStatus,
5158    detail: Option<&str>,
5159    action: HealthAction,
5160    now_ms: u64,
5161) {
5162    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5163    match action {
5164        HealthAction::Report => {
5165            info!(
5166                module_id = %spec.module_id,
5167                status = ?status,
5168                detail,
5169                "module reported non-ok health"
5170            );
5171        }
5172        HealthAction::Alert => {
5173            error!(
5174                module_id = %spec.module_id,
5175                status = ?status,
5176                detail,
5177                "module health alert"
5178            );
5179        }
5180        HealthAction::Restart => {
5181            if let Err(err) = health_restart_child(
5182                spec,
5183                runtime,
5184                registry,
5185                process_liveness,
5186                snapshot,
5187                child,
5188                status,
5189                detail,
5190                now_ms,
5191            )
5192            .await
5193            {
5194                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5195            }
5196        }
5197    }
5198}
5199
5200#[allow(clippy::too_many_arguments)]
5201async fn health_restart_child(
5202    spec: &ModuleSpec,
5203    runtime: &SupervisorRuntimeConfig,
5204    registry: &Registry,
5205    process_liveness: &SupervisorProcessLiveness,
5206    snapshot: &SharedSnapshot,
5207    child: &mut Option<SupervisedChild>,
5208    status: SupervisorHealthStatus,
5209    detail: Option<&str>,
5210    now_ms: u64,
5211) -> Result<(), SuperviseError> {
5212    let (enabled, schedule) = {
5213        let mut state = lock_snapshot(snapshot)?;
5214        let enabled = state.enabled;
5215        let schedule = if enabled {
5216            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5217        } else {
5218            None
5219        };
5220        (enabled, schedule)
5221    };
5222
5223    if !enabled {
5224        return Err(SuperviseError::Disabled {
5225            module_id: spec.module_id.clone(),
5226        });
5227    }
5228
5229    if schedule.is_none() {
5230        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5231        error!(
5232            module_id = %spec.module_id,
5233            status = ?status,
5234            detail,
5235            max_restarts = runtime.restart_policy.max_restarts,
5236            window_secs = runtime.restart_policy.window.as_secs(),
5237            reason = %runtime.restart_policy.budget_exhausted_detail(),
5238            "health restart budget exhausted; marking module failed"
5239        );
5240        let stop_notice = begin_forwarding_drain_if_configured(
5241            spec,
5242            runtime,
5243            registry,
5244            snapshot,
5245            Some(true),
5246            RouteCloseReason::Disable,
5247        )
5248        .await?;
5249        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5250            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5251        })?;
5252        drain_optional_child(
5253            &spec.module_id,
5254            spec.protocol,
5255            stop_notice,
5256            registry,
5257            runtime.forwarding.as_deref(),
5258            snapshot,
5259            &runtime.terminal_ring,
5260            &runtime.spawn_events,
5261            child,
5262            runtime.drain_timeout,
5263            ModuleState::Failed,
5264            Some(true),
5265        )
5266        .await?;
5267        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5268        return Ok(());
5269    }
5270
5271    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5272    let mut restart_count = 0;
5273    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5274        restart_count = state.crash_restarts.len();
5275        state.state = ModuleState::Unresponsive;
5276        state.health.status = status;
5277        state.health.last_action = Some(HealthAction::Restart.to_string());
5278        state.health.last_action_ms = Some(now_ms);
5279    })?;
5280    warn!(
5281        module_id = %spec.module_id,
5282        status = ?status,
5283        detail,
5284        restart_count,
5285        restart_in_window = schedule.restart_in_window,
5286        delay_ms = schedule.delay.as_millis() as u64,
5287        "health-triggered module restart"
5288    );
5289
5290    let stop_notice = begin_forwarding_drain_if_configured(
5291        spec,
5292        runtime,
5293        registry,
5294        snapshot,
5295        Some(true),
5296        RouteCloseReason::Restart,
5297    )
5298    .await?;
5299    drain_optional_child(
5300        &spec.module_id,
5301        spec.protocol,
5302        stop_notice,
5303        registry,
5304        runtime.forwarding.as_deref(),
5305        snapshot,
5306        &runtime.terminal_ring,
5307        &runtime.spawn_events,
5308        child,
5309        runtime.drain_timeout,
5310        ModuleState::Restarting,
5311        Some(true),
5312    )
5313    .await?;
5314    schedule_respawn(
5315        runtime,
5316        snapshot,
5317        &spec.module_id,
5318        schedule.delay,
5319        RespawnKind::Spawn,
5320    )
5321}
5322
5323fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5324    if let Some(reply) = runtime
5325        .deferred_reload_reply
5326        .lock()
5327        .unwrap_or_else(|p| p.into_inner())
5328        .take()
5329    {
5330        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5331            module_id: module_id.to_string(),
5332            reason: reason.to_string(),
5333        }));
5334    }
5335}
5336
5337fn schedule_respawn(
5338    runtime: &SupervisorRuntimeConfig,
5339    snapshot: &SharedSnapshot,
5340    module_id: &str,
5341    delay: Duration,
5342    kind: RespawnKind,
5343) -> Result<(), SuperviseError> {
5344    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5345    update_snapshot(snapshot, Some(module_id), |state| {
5346        state.respawn_pending = true
5347    })?;
5348    *runtime
5349        .scheduled_respawn
5350        .lock()
5351        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5352        deadline: Instant::now() + delay,
5353        kind,
5354    });
5355    Ok(())
5356}
5357
5358fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5359    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5360        state.health.last_action = Some(action);
5361        state.health.last_action_ms = Some(now_ms);
5362    });
5363}
5364
5365fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5366    match status {
5367        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5368        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5369        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5370    }
5371}
5372
5373/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5374/// returned to every `supervisor.list` and `supervisor.health` caller.
5375///
5376/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5377/// path: that request exists to return a module's complete metrics object, and
5378/// `ck health <module-id>` documents it as the way to see what the cached view
5379/// truncates. The asymmetry is the feature.
5380///
5381/// So a new caller must decide which side it is on rather than assume the cap is
5382/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5383/// the truncation that path exists to avoid.
5384fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5385    let metrics = metrics?;
5386    match serde_json::to_vec(&metrics) {
5387        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5388            "truncated": true,
5389            "original_bytes": encoded.len(),
5390        })),
5391        Ok(_) | Err(_) => Some(metrics),
5392    }
5393}
5394
5395/// Spread health probes so a fleet-wide restart does not converge them.
5396///
5397/// The delay is derived from the module id and probe index rather than a random
5398/// source, so it is deterministic per module: a module keeps its own offset
5399/// across daemon restarts instead of re-rolling into a collision.
5400fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5401    if cadence.is_zero() {
5402        return Duration::ZERO;
5403    }
5404    let cadence_ms = cadence.as_millis() as u64;
5405    // This early return is REDUNDANT, deliberately, and a mutation run will show
5406    // it surviving removal. Recording why here so the next person to notice does
5407    // not have to re-derive it:
5408    //
5409    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5410    //   a zero cadence and builds the Duration from whole milliseconds, so a
5411    //   sub-millisecond cadence cannot come from config.
5412    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5413    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5414    //   -- exactly what this returns.
5415    //
5416    // Kept as a guard against a future widening of the config parser (accepting
5417    // microseconds, say), which would make the sub-millisecond case reachable.
5418    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5419    // divides by zero. Remove this and nothing changes.
5420    if cadence_ms == 0 {
5421        return cadence;
5422    }
5423    // Note that this never returns less than one cadence, including for the FIRST
5424    // probe. So a freshly registered module reports health `unknown` for a full
5425    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5426    // ready to answer.
5427    //
5428    // That is a property of the supervisor's schedule, not of any module: an
5429    // operator watching a restart sees `unknown` and cannot tell it from a module
5430    // that is slow to warm. Measured on two unrelated modules, both flipping to
5431    // `ok` between 22s and 32s after restart.
5432    //
5433    // Left as-is because spreading the first probe is what keeps a fleet-wide
5434    // restart from firing fourteen simultaneous probes into a cold machine. The
5435    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5436    // that thundering herd for a faster first reading.
5437    let jitter_span = (cadence_ms / 10).max(1);
5438    let hash = module_id.as_bytes().iter().fold(
5439        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5440        |acc, byte| {
5441            acc.wrapping_mul(1099511628211)
5442                .wrapping_add(u64::from(*byte))
5443        },
5444    );
5445    cadence + Duration::from_millis(hash % jitter_span)
5446}
5447
5448#[cfg(test)]
5449mod tests {
5450    use super::*;
5451
5452    #[test]
5453    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5454        let handle = SupervisorHandle::new();
5455        let module_id = "readded-tombstone";
5456        handle.record_rescan_removal(module_id);
5457        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5458
5459        handle.apply_identity_configuration(&ModuleSpec {
5460            module_id: module_id.to_string(),
5461            program: PathBuf::from("/test/module"),
5462            args: Vec::new(),
5463            env: Vec::new(),
5464            reserved: false,
5465            reserved_prefixes: Vec::new(),
5466            protocol: ModuleProtocol::Subc,
5467            overlap: Default::default(),
5468        });
5469
5470        assert!(
5471            handle.removal_tombstone_age_ms(module_id).is_none(),
5472            "a re-added module must not retain a stale removal tombstone"
5473        );
5474    }
5475
5476    /// What one module's owner looked like from the control plane at the
5477    /// instant after its first process was spawned.
5478    #[derive(Debug, PartialEq, Eq)]
5479    struct OwnerInSpawnWindow {
5480        module_id: String,
5481        configured: bool,
5482        on_roster: bool,
5483        admission_refusal: Option<&'static str>,
5484    }
5485
5486    /// A supervised module's process can connect, register, sync its scopes
5487    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5488    /// the module on the roster until afterward; configured supervision does
5489    /// so before spawning. In both cases the owner must
5490    /// already read as configured, so a scoped `route.open` against it is
5491    /// refused as retryable `scope_not_synced` and not as terminal
5492    /// `scope_not_live` ("will never sync").
5493    ///
5494    /// The hook runs just after each initial spawn attempt, before returning to
5495    /// the caller, so no race with a real child's registration is needed.
5496    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5497    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5498        use crate::scopes::ScopeTable;
5499        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5500
5501        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5502        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5503            module_id: module_id.to_string(),
5504            program,
5505            args: Vec::new(),
5506            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5507                .into_iter()
5508                .map(|key| (key.to_string(), dir.path().display().to_string()))
5509                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5510                .collect(),
5511            reserved: false,
5512            reserved_prefixes: Vec::new(),
5513            protocol: ModuleProtocol::Subc,
5514            overlap: Default::default(),
5515        };
5516        let live = super::terminal_history_tests::fake_aft_stub_path();
5517        let missing = dir.path().join("definitely-missing-module");
5518
5519        let handle = SupervisorHandle::new();
5520        let mut supervisor =
5521            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5522                .with_handle(handle.clone());
5523        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5524        let hook_handle = handle.clone();
5525        let hook_observed = Arc::clone(&observed);
5526        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5527            // Exactly what the control plane computes for a scoped route.open
5528            // naming this module as the owner of a scope it has not synced.
5529            let configured = hook_handle.is_configured(module_id);
5530            let selector = ScopeSelector::new(
5531                Principal::Reserved {
5532                    module_id: module_id.to_string(),
5533                },
5534                "s".to_string(),
5535                Some(1),
5536            );
5537            let carrier = Principal::Reserved {
5538                module_id: "carrier".to_string(),
5539            };
5540            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5541                .admit(&carrier, module_id, &selector, configured)
5542            {
5543                Ok(_) => None,
5544                Err(refusal) => Some(refusal.code),
5545            };
5546            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5547                module_id: module_id.to_string(),
5548                configured,
5549                on_roster: hook_handle.get(module_id).is_some(),
5550                admission_refusal,
5551            });
5552        })));
5553
5554        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5555        let configured = supervisor
5556            .supervise_configured(stub("configured", live.clone()), true)
5557            .unwrap();
5558        let with_health = supervisor
5559            .supervise_configured_with_health(
5560                stub("with-health", live.clone()),
5561                true,
5562                HealthConfig::default(),
5563                None,
5564                RestartPolicy::default(),
5565            )
5566            .unwrap();
5567        // The failed-spawn path still puts the module on the roster (as
5568        // failed), so it is configured throughout.
5569        let failed = supervisor
5570            .supervise_configured_with_health(
5571                stub("failed-spawn", missing.clone()),
5572                true,
5573                HealthConfig::default(),
5574                None,
5575                RestartPolicy::default(),
5576            )
5577            .unwrap();
5578        // A failed plain `spawn` puts nothing on the roster, so its mark is
5579        // taken back once the spawn has failed.
5580        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5581
5582        let expected = [
5583            "plain",
5584            "configured",
5585            "with-health",
5586            "failed-spawn",
5587            "spawn-error",
5588        ]
5589        .into_iter()
5590        .map(|module_id| OwnerInSpawnWindow {
5591            module_id: module_id.to_string(),
5592            configured: true,
5593            on_roster: !matches!(module_id, "plain" | "spawn-error"),
5594            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5595        })
5596        .collect::<Vec<_>>();
5597        assert_eq!(*observed.lock().unwrap(), expected);
5598
5599        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5600            assert!(
5601                handle.get(module_id).is_some(),
5602                "{module_id} is on the roster"
5603            );
5604            assert!(
5605                handle.is_configured(module_id),
5606                "{module_id} stays configured"
5607            );
5608        }
5609        assert!(handle.get("spawn-error").is_none());
5610        assert!(
5611            !handle.is_configured("spawn-error"),
5612            "a plain spawn that failed must not leave its module marked configured"
5613        );
5614
5615        // Leaving the roster clears the mark with it.
5616        handle.retire("failed-spawn");
5617        assert!(!handle.is_configured("failed-spawn"));
5618
5619        for module in [plain, configured, with_health] {
5620            module.stop().await.unwrap();
5621        }
5622        drop(failed);
5623    }
5624
5625    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5626        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5627        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5628            snapshot.process_alive = true;
5629            snapshot.pid = Some(41);
5630            snapshot.spawned_at_ms = Some(42);
5631            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5632            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5633                device: 43,
5634                inode: 44,
5635                #[cfg(windows)]
5636                windows_image: None,
5637            });
5638        })
5639        .unwrap();
5640        snapshot
5641    }
5642
5643    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5644        let snapshot = lock_snapshot(snapshot).unwrap();
5645        assert!(!snapshot.process_alive);
5646        assert_eq!(snapshot.pid, None);
5647        assert_eq!(snapshot.spawned_at_ms, None);
5648        assert_eq!(snapshot.spawned_from, None);
5649        assert_eq!(snapshot.spawned_file_identity, None);
5650    }
5651
5652    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5653    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5654        let supervisor =
5655            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5656        let mut runtime = supervisor.runtime_config();
5657        runtime.test_seed_stale_facts_before_enable_spawn = true;
5658        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5659        let mut child = None;
5660        let spec = ModuleSpec {
5661            module_id: "failed-enable-clears-facts".to_string(),
5662            program: PathBuf::from("/definitely/missing/failed-enable-module"),
5663            args: Vec::new(),
5664            env: Vec::new(),
5665            reserved: false,
5666            reserved_prefixes: Vec::new(),
5667            protocol: ModuleProtocol::Subc,
5668            overlap: Default::default(),
5669        };
5670
5671        let result = set_child_enabled(
5672            &spec,
5673            &runtime,
5674            &supervisor.registry,
5675            &supervisor.process_liveness,
5676            &snapshot,
5677            &mut child,
5678            true,
5679        )
5680        .await;
5681
5682        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5683        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5684        assert_snapshot_process_facts_cleared(&snapshot);
5685    }
5686
5687    #[tokio::test]
5688    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5689        let supervisor =
5690            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5691        let runtime = supervisor.runtime_config();
5692        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5693            ModuleState::Restarting,
5694            true,
5695        )));
5696        let spec = ModuleSpec {
5697            module_id: "start-stranded-restarting".to_string(),
5698            program: super::terminal_history_tests::fake_aft_stub_path(),
5699            args: Vec::new(),
5700            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5701            reserved: false,
5702            reserved_prefixes: Vec::new(),
5703            protocol: ModuleProtocol::None,
5704            overlap: Default::default(),
5705        };
5706        let mut child = None;
5707        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5708        assert!(!super::set_child_enabled(
5709            &spec,
5710            &runtime,
5711            &Registry::default(),
5712            &supervisor.process_liveness,
5713            &snapshot,
5714            &mut child,
5715            true
5716        )
5717        .await
5718        .unwrap());
5719        assert!(child.is_none());
5720        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5721        assert!(super::set_child_enabled(
5722            &spec,
5723            &runtime,
5724            &Registry::default(),
5725            &supervisor.process_liveness,
5726            &snapshot,
5727            &mut child,
5728            true
5729        )
5730        .await
5731        .unwrap());
5732        assert_eq!(
5733            lock_snapshot(&snapshot).unwrap().state,
5734            ModuleState::Running
5735        );
5736        let mut child = child.unwrap();
5737        child.start_kill().unwrap();
5738        child.wait().await.unwrap();
5739    }
5740
5741    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5742    async fn failed_reload_spawn_clears_current_process_facts() {
5743        let supervisor =
5744            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5745        let mut runtime = supervisor.runtime_config();
5746        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5747        let snapshot = stale_process_snapshot(ModuleState::Running, true);
5748        let mut child = None;
5749        let spec = ModuleSpec {
5750            module_id: "failed-reload-clears-facts".to_string(),
5751            program: PathBuf::from("/unused/failed-reload-module"),
5752            args: Vec::new(),
5753            env: Vec::new(),
5754            reserved: false,
5755            reserved_prefixes: Vec::new(),
5756            protocol: ModuleProtocol::Subc,
5757            overlap: Default::default(),
5758        };
5759
5760        let result = handle_reload_spawn_failure(
5761            &spec,
5762            &runtime,
5763            &supervisor.process_liveness,
5764            &snapshot,
5765            &mut child,
5766            "forced reload spawn failure".to_string(),
5767        )
5768        .await;
5769
5770        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5771        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5772        assert_snapshot_process_facts_cleared(&snapshot);
5773    }
5774
5775    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5776    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5777        let supervisor =
5778            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5779        let snapshot = stale_process_snapshot(ModuleState::Running, true);
5780        let module = supervisor.supervised_module(
5781            ModuleSpec {
5782                module_id: "drop-clears-facts".to_string(),
5783                program: PathBuf::from("/unused/drop-module"),
5784                args: Vec::new(),
5785                env: Vec::new(),
5786                reserved: false,
5787                reserved_prefixes: Vec::new(),
5788                protocol: ModuleProtocol::Subc,
5789                overlap: Default::default(),
5790            },
5791            supervisor.runtime_config(),
5792            Arc::clone(&snapshot),
5793            None,
5794        );
5795        assert!(!module
5796            .inner
5797            .monitor
5798            .lock()
5799            .unwrap()
5800            .as_ref()
5801            .unwrap()
5802            .is_finished());
5803
5804        drop(module);
5805
5806        assert_eq!(
5807            lock_snapshot(&snapshot).unwrap().state,
5808            ModuleState::Stopped
5809        );
5810        assert_snapshot_process_facts_cleared(&snapshot);
5811    }
5812
5813    #[cfg(unix)]
5814    #[tokio::test]
5815    async fn rescan_preserves_running_protocol_until_respawn() {
5816        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5817        let initial = ModuleSpec {
5818            module_id: "rescan-protocol".into(),
5819            program: PathBuf::from("/bin/sleep"),
5820            args: vec!["60".into()],
5821            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5822                .into_iter()
5823                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
5824                .collect(),
5825            reserved: false,
5826            reserved_prefixes: vec![],
5827            protocol: ModuleProtocol::None,
5828            overlap: Default::default(),
5829        };
5830        let supervisor =
5831            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5832        let module = supervisor.spawn(initial.clone()).unwrap();
5833        assert!(module.status().unwrap().live);
5834        let mut next = initial;
5835        next.protocol = ModuleProtocol::Subc;
5836        module
5837            .update_configuration(next.clone(), HealthConfig::default(), None)
5838            .await
5839            .unwrap();
5840        assert!(
5841            module.status().unwrap().live,
5842            "rescan must not require HELLO from the old non-wire process"
5843        );
5844        let runtime = supervisor.runtime_config();
5845        let action = on_child_exit(
5846            &next,
5847            RestartPolicy::default(),
5848            &supervisor.registry,
5849            &module.inner.snapshot,
5850            &runtime.terminal_ring,
5851            &runtime.spawn_events,
5852            &runtime.child_roster,
5853            ExitReport {
5854                kind: ExitKind::Clean,
5855                code: Some(0),
5856                signal: None,
5857                at_ms: unix_ms_now(),
5858            },
5859        )
5860        .await;
5861        assert!(
5862            matches!(action, NextAction::Restart { .. }),
5863            "the old non-wire process's clean exit must restart"
5864        );
5865        module.drain().await.unwrap();
5866    }
5867
5868    #[cfg(unix)]
5869    #[tokio::test]
5870    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
5871        use std::os::unix::fs::PermissionsExt;
5872        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
5873        let script = dir.join("module.sh");
5874        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
5875        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
5876        let record_path = dir.join("live-children.json");
5877        let supervisor =
5878            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5879                .with_live_children_record(&record_path);
5880        for (program, args) in [
5881            (PathBuf::from("sleep"), vec!["60".into()]),
5882            (script, vec![]),
5883        ] {
5884            let spec = ModuleSpec {
5885                module_id: "image-identity".into(),
5886                program: program.clone(),
5887                args,
5888                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5889                    .into_iter()
5890                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
5891                    .collect(),
5892                reserved: false,
5893                reserved_prefixes: vec![],
5894                protocol: ModuleProtocol::None,
5895                overlap: Default::default(),
5896            };
5897            let module = supervisor.spawn(spec).unwrap();
5898            #[cfg(target_os = "macos")]
5899            {
5900                // SETEXEC confirmation is asynchronous; the orphan record must
5901                // identify the final image, never the intermediate trampoline.
5902                let deadline = Instant::now() + Duration::from_secs(5);
5903                while crate::live_children::read_record(&record_path)
5904                    .unwrap()
5905                    .iter()
5906                    .all(|entry| entry.executable.is_none())
5907                {
5908                    assert!(Instant::now() < deadline, "module image was not confirmed");
5909                    tokio::time::sleep(Duration::from_millis(5)).await;
5910                }
5911            }
5912            let entry = crate::live_children::read_record(&record_path)
5913                .unwrap()
5914                .pop()
5915                .unwrap();
5916            let observed = subc_os::Process::open(entry.pid)
5917                .unwrap()
5918                .unwrap()
5919                .observe()
5920                .unwrap();
5921            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
5922            module.drain().await.unwrap();
5923            assert_eq!(
5924                verdict,
5925                crate::live_children::IdentityVerdict::Matches,
5926                "program {program:?}: recorded {entry:?}, observed {observed:?}"
5927            );
5928        }
5929    }
5930
5931    #[cfg(unix)]
5932    fn http_fixture(
5933        dir: &std::path::Path,
5934        url: &str,
5935        threshold: u32,
5936    ) -> crate::daemon_config::ConfiguredModule {
5937        let path = dir.join("subc.jsonc");
5938        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
5939            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
5940            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
5941            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
5942            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
5943        }}}).to_string()).unwrap();
5944        crate::daemon_config::load(&path)
5945            .unwrap()
5946            .unwrap()
5947            .modules
5948            .pop()
5949            .unwrap()
5950    }
5951
5952    #[cfg(unix)]
5953    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
5954        timeout(Duration::from_secs(5), async {
5955            loop {
5956                if module.status().unwrap().health.status == status {
5957                    break;
5958                }
5959                sleep(Duration::from_millis(5)).await;
5960            }
5961        })
5962        .await
5963        .unwrap_or_else(|_| {
5964            panic!(
5965                "expected {status:?}, got {:?}",
5966                module.status().unwrap().health
5967            )
5968        });
5969    }
5970
5971    #[cfg(unix)]
5972    #[tokio::test]
5973    async fn http_health_status_flips_ok_failing_ok() {
5974        use tokio::io::{AsyncReadExt, AsyncWriteExt};
5975        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
5976        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
5977        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
5978        let serving_status = status.clone();
5979        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
5980        let server = tokio::spawn(async move {
5981            loop {
5982                let (mut stream, _) = listener.accept().await.unwrap();
5983                let mut request = [0u8; 2048];
5984                let count = stream.read(&mut request).await.unwrap();
5985                assert!(count > 0, "a probe must send an HTTP request");
5986                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
5987                let body = if code == 200 {
5988                    "ready"
5989                } else {
5990                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
5991                };
5992                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
5993                let _ = stream.write_all(response.as_bytes()).await;
5994            }
5995        });
5996        let configured = http_fixture(&dir, &url, 1000);
5997        let module =
5998            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5999                .supervise_configured_with_health(
6000                    configured.module_spec(),
6001                    true,
6002                    configured.health,
6003                    configured.drain_timeout_ms,
6004                    configured.restart,
6005                )
6006                .unwrap();
6007        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6008        status.store(503, std::sync::atomic::Ordering::SeqCst);
6009        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6010        assert!(module
6011            .status()
6012            .unwrap()
6013            .health
6014            .detail
6015            .unwrap()
6016            .contains("scratch failure"));
6017        status.store(200, std::sync::atomic::Ordering::SeqCst);
6018        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6019        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6020        let before = module.status().unwrap();
6021        let (spec, mut health) = module.configuration().unwrap();
6022        health.http = None;
6023        module
6024            .update_configuration(spec.clone(), health.clone(), Some(10))
6025            .await
6026            .unwrap();
6027        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6028        health.http = Some(url);
6029        module
6030            .update_configuration(spec, health, Some(10))
6031            .await
6032            .unwrap();
6033        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6034        assert_eq!(
6035            module.status().unwrap().pid,
6036            before.pid,
6037            "changing a probe must apply live, not restart its process"
6038        );
6039        let (spec, mut health) = module.configuration().unwrap();
6040        health.failure_threshold = 2;
6041        module
6042            .update_configuration(spec, health, Some(10))
6043            .await
6044            .unwrap();
6045        status.store(503, std::sync::atomic::Ordering::SeqCst);
6046        timeout(Duration::from_secs(5), async {
6047            while module.status().unwrap().spawn_generation == before.spawn_generation {
6048                sleep(Duration::from_millis(5)).await;
6049            }
6050        })
6051        .await
6052        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6053        module.drain().await.unwrap();
6054        server.abort();
6055    }
6056
6057    #[cfg(unix)]
6058    #[tokio::test]
6059    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6060        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6061        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6062        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6063        drop(unused);
6064        let configured = http_fixture(&dir, &url, 2);
6065        let module =
6066            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6067                .supervise_configured_with_health(
6068                    configured.module_spec(),
6069                    true,
6070                    configured.health,
6071                    configured.drain_timeout_ms,
6072                    configured.restart,
6073                )
6074                .unwrap();
6075        let before = module.status().unwrap().spawn_generation;
6076        timeout(Duration::from_secs(5), async {
6077            loop {
6078                let status = module.status().unwrap();
6079                if status.spawn_generation > before {
6080                    assert!(status.lifetime_restarts > 0);
6081                    break;
6082                }
6083                sleep(Duration::from_millis(5)).await;
6084            }
6085        })
6086        .await
6087        .expect("sustained HTTP refusal must trigger the health restart policy");
6088        module.drain().await.unwrap();
6089    }
6090
6091    #[cfg(unix)]
6092    #[tokio::test]
6093    async fn http_health_timeout_honours_deadline() {
6094        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6095        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6096        let server = tokio::spawn(async move {
6097            let _held = listener.accept().await.unwrap();
6098            std::future::pending::<()>().await;
6099        });
6100        let error = timeout(
6101            Duration::from_secs(1),
6102            probe_http_health(&url, Duration::from_millis(10)),
6103        )
6104        .await
6105        .expect("the probe must enforce its own deadline")
6106        .unwrap_err();
6107        server.abort();
6108        assert!(error.to_string().contains("timed out"));
6109    }
6110
6111    #[cfg(unix)]
6112    #[tokio::test]
6113    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6114        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6115        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6116        let url = format!(
6117            "http://localhost:{}/healthz",
6118            listener.local_addr().unwrap().port()
6119        );
6120        let server = tokio::spawn(async move {
6121            let (mut stream, _) = listener.accept().await.unwrap();
6122            let mut request = [0u8; 2048];
6123            assert!(stream.read(&mut request).await.unwrap() > 0);
6124            stream
6125                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6126                .await
6127                .unwrap();
6128        });
6129        // The deadline only bounds a hang. A probe that never tried the IPv6
6130        // address would be refused on 127.0.0.1 and fail at once, so a longer
6131        // deadline does not weaken the assertion; one second timed out under a
6132        // loaded parallel test run.
6133        assert_eq!(
6134            probe_http_health(&url, Duration::from_secs(10))
6135                .await
6136                .unwrap()
6137                .status,
6138            HealthStatus::Ok
6139        );
6140        server.await.unwrap();
6141    }
6142
6143    #[cfg(unix)]
6144    #[tokio::test]
6145    async fn http_health_timeout_keeps_partial_status_and_body() {
6146        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6147        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6148        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6149        let server = tokio::spawn(async move {
6150            let (mut stream, _) = listener.accept().await.unwrap();
6151            let mut request = [0u8; 2048];
6152            assert!(stream.read(&mut request).await.unwrap() > 0);
6153            stream
6154                .write_all(
6155                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6156                )
6157                .await
6158                .unwrap();
6159            std::future::pending::<()>().await;
6160        });
6161        let error = probe_http_health(&url, Duration::from_secs(1))
6162            .await
6163            .unwrap_err()
6164            .to_string();
6165        server.abort();
6166        assert!(
6167            error.contains("timed out")
6168                && error.contains("503 Unavailable")
6169                && error.contains("partial diagnostic"),
6170            "{error}"
6171        );
6172    }
6173
6174    #[cfg(unix)]
6175    #[tokio::test]
6176    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6177        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6178        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6179        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6180        let server = tokio::spawn(async move {
6181            let (mut stream, _) = listener.accept().await.unwrap();
6182            let mut request = [0u8; 2048];
6183            assert!(stream.read(&mut request).await.unwrap() > 0);
6184            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6185            let response = format!(
6186                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6187                body.len()
6188            );
6189            stream.write_all(response.as_bytes()).await.unwrap();
6190        });
6191        let error = probe_http_health(&url, Duration::from_secs(1))
6192            .await
6193            .unwrap_err()
6194            .to_string();
6195        server.await.unwrap();
6196        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6197        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6198        assert!(!error.contains("not-in-diagnostic"));
6199    }
6200
6201    #[cfg(unix)]
6202    #[tokio::test]
6203    async fn http_health_real_nats_server_monitoring() {
6204        if std::process::Command::new("nats-server")
6205            .arg("--version")
6206            .env("XDG_DATA_HOME", std::env::temp_dir())
6207            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6208            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6209            .output()
6210            .is_err()
6211        {
6212            eprintln!(
6213                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6214            );
6215            return;
6216        }
6217        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6218        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6219        let port = monitor.local_addr().unwrap().port();
6220        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6221        let client_port = client.local_addr().unwrap().port();
6222        let config = dir.join("server.conf");
6223        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6224        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6225        configured.program = PathBuf::from("nats-server");
6226        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6227        drop(monitor);
6228        drop(client);
6229        let module =
6230            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6231                .supervise_configured_with_health(
6232                    configured.module_spec(),
6233                    true,
6234                    configured.health,
6235                    configured.drain_timeout_ms,
6236                    configured.restart,
6237                )
6238                .unwrap();
6239        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6240        module.drain().await.unwrap();
6241    }
6242
6243    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6244    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6245        let supervisor =
6246            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6247        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6248        let initial = ModuleSpec {
6249            module_id: "rescan-preserves-spawn-facts".to_string(),
6250            program: PathBuf::from("/spawned/module"),
6251            args: Vec::new(),
6252            env: Vec::new(),
6253            reserved: false,
6254            reserved_prefixes: Vec::new(),
6255            protocol: ModuleProtocol::Subc,
6256            overlap: Default::default(),
6257        };
6258        let module = supervisor.supervised_module(
6259            initial.clone(),
6260            supervisor.runtime_config(),
6261            snapshot,
6262            None,
6263        );
6264        let before = module.status().unwrap();
6265        let mut replacement = initial;
6266        replacement.program = PathBuf::from("/rescanned/replacement-module");
6267
6268        module
6269            .update_configuration(replacement, HealthConfig::default(), None)
6270            .await
6271            .unwrap();
6272
6273        let after = module.status().unwrap();
6274        assert_eq!(after.pid, before.pid);
6275        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6276        assert_eq!(after.spawned_from, before.spawned_from);
6277        drop(module);
6278    }
6279}
6280
6281fn unix_ms_now() -> u64 {
6282    SystemTime::now()
6283        .duration_since(UNIX_EPOCH)
6284        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6285        .unwrap_or(0)
6286}
6287
6288async fn supervise_loop(
6289    mut spec: ModuleSpec,
6290    mut runtime: SupervisorRuntimeConfig,
6291    registry: Arc<Registry>,
6292    process_liveness: Arc<SupervisorProcessLiveness>,
6293    snapshot: SharedSnapshot,
6294    mut child: Option<SupervisedChild>,
6295    mut commands: mpsc::Receiver<SupervisorCommand>,
6296) {
6297    let mut health_probe = HealthProbeRuntime::default();
6298    // Registry writes (including embedded callers) notify this module only.
6299    // Subscribe before the first refresh; watch retains changes that arrive
6300    // while commands or probes are running. No polling fallback is needed.
6301    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6302        Ok(changes) => Some(changes),
6303        Err(err) => {
6304            // A poisoned registry cannot accept further writes, so it cannot
6305            // recover via a timer. Keep exit and command handling alive.
6306            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6307            None
6308        }
6309    };
6310    // All restart backoffs run here, including health and operator requests.
6311    // While one is pending the loop serves commands, so disable or drain can
6312    // cancel the replacement without spawning a process just to stop it.
6313    let mut pending_respawn: Option<PendingRespawn> = None;
6314    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
6315    // before anything else so a stop that interrupted a swap runs at once.
6316    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6317    let mut previous_instance_recheck = tokio::time::interval_at(
6318        Instant::now() + Duration::from_secs(5),
6319        Duration::from_secs(5),
6320    );
6321    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6322    loop {
6323        #[cfg(test)]
6324        {
6325            lock_snapshot(&snapshot).unwrap().actor_select = None;
6326        }
6327        #[cfg(target_os = "macos")]
6328        if let Some(active) = child.as_mut() {
6329            active.confirm_privacy_exec().await;
6330        }
6331        if let Some(scheduled) = runtime
6332            .scheduled_respawn
6333            .lock()
6334            .unwrap_or_else(|p| p.into_inner())
6335            .take()
6336        {
6337            pending_respawn = Some(scheduled);
6338        }
6339        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6340            pending_respawn = None;
6341            cancel_deferred_reload(
6342                &runtime,
6343                &spec.module_id,
6344                "respawn cancelled by a supervisor command",
6345            );
6346        }
6347        if child.is_none()
6348            && pending_respawn.is_none()
6349            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6350        {
6351            cancel_deferred_reload(
6352                &runtime,
6353                &spec.module_id,
6354                "respawn cancelled before a replacement was spawned",
6355            );
6356            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6357                state.respawn_pending = false;
6358                state.coalesced_restart_pending = false;
6359                if matches!(
6360                    state.state,
6361                    ModuleState::Restarting
6362                        | ModuleState::Starting
6363                        | ModuleState::Draining
6364                        | ModuleState::Unresponsive
6365                ) {
6366                    error!(module_id = %spec.module_id, state = ?state.state,
6367                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6368                    state.state = ModuleState::Failed;
6369                    clear_current_process_facts(state);
6370                }
6371            });
6372        }
6373        if let Some(command) = requeued.pop_front() {
6374            if !handle_supervisor_command(
6375                command,
6376                &mut spec,
6377                &mut runtime,
6378                &registry,
6379                &process_liveness,
6380                &snapshot,
6381                &mut child,
6382                &mut commands,
6383                &mut requeued,
6384            )
6385            .await
6386            {
6387                return;
6388            }
6389            if child.is_some() || !respawn_still_pending(&snapshot) {
6390                pending_respawn = None;
6391                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6392                    state.respawn_pending = false
6393                });
6394            }
6395            continue;
6396        }
6397        if child.is_some() {
6398            #[cfg(test)]
6399            {
6400                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6401            }
6402            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
6403            let wake_after = health_probe.wake_after();
6404            let probe_sleep = async move {
6405                match wake_after {
6406                    Some(delay) => sleep(delay).await,
6407                    None => std::future::pending().await,
6408                }
6409            };
6410            tokio::pin!(probe_sleep);
6411            // A `protocol: "none"` child (a plain process that never registers
6412            // over the subc wire, such as nats-server) with no HTTP health
6413            // check has nothing to probe, so it waits only for its exit or a
6414            // supervisor command. A rescan that adds an HTTP check is a command.
6415            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6416            #[cfg(test)]
6417            {
6418                let registration_pending = wire_child
6419                    && registration_changes
6420                        .as_ref()
6421                        .is_some_and(|changes| changes.has_changed().unwrap());
6422                let mut state = lock_snapshot(&snapshot).unwrap();
6423                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6424                    generation: state.spawn_generation,
6425                    turn: state.actor_turns,
6426                    registered_connection: health_probe.registered_connection,
6427                    next_probe_at: health_probe.next_probe_at,
6428                    wake_after,
6429                });
6430            }
6431            let active_child = child.as_mut().expect("child checked above");
6432            tokio::select! {
6433                wait_result = active_child.wait() => {
6434                    #[cfg(test)]
6435                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6436                    // Every arm below that gives up on the CHILD must keep the
6437                    // supervision task itself alive (child = None, loop
6438                    // continues into command-serving mode). Returning here
6439                    // closes the command channel, which makes the module
6440                    // permanently unrestartable in-band: a clean child exit
6441                    // of an enabled module once wedged the fleet this way
6442                    // ('supervisor command channel is closed') and required a
6443                    // full daemon restart to recover.
6444                    let exit_report = match wait_result {
6445                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6446                        Err(err) => {
6447                            active_child.drain_stderr(&spec.module_id).await;
6448                            fail_after_wait_error(
6449                                &spec.module_id,
6450                                &snapshot,
6451                                &runtime.terminal_ring,
6452                                &runtime.spawn_events,
6453                                format!("failed to wait for supervised module: {err}"),
6454                            );
6455                            untrack_if_registration_released(
6456                                &process_liveness,
6457                                &registry,
6458                                &spec.module_id,
6459                                &snapshot,
6460                            );
6461                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6462                            child = None;
6463                            continue;
6464                        }
6465                    };
6466                    active_child.drain_stderr(&spec.module_id).await;
6467
6468                    let next = on_child_exit(
6469                        &spec,
6470                        runtime.restart_policy,
6471                        &registry,
6472                        &snapshot,
6473                        &runtime.terminal_ring,
6474                        &runtime.spawn_events,
6475                        &runtime.child_roster,
6476                        exit_report,
6477                    ).await;
6478                    runtime.start_requester = None;
6479                    runtime.pending_activity_restart = false;
6480                    // The exit is recorded, so a daemon shutdown may stop
6481                    // waiting for this child (see `SupervisedChild::wait`).
6482                    active_child.release_roster();
6483                    match next {
6484                        NextAction::Stop { registration_released } => {
6485                            if registration_released {
6486                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6487                            }
6488                            child = None;
6489                        }
6490                        NextAction::Restart { schedule } => {
6491                            let delay = schedule.map_or(
6492                                runtime.restart_policy.delay_for_restart(0),
6493                                |schedule| schedule.delay,
6494                            );
6495                            if let Some(schedule) = schedule {
6496                                log_crash_respawn(&spec.module_id, schedule);
6497                            }
6498                            // The exited child is fully recorded at this point,
6499                            // so release it and count the backoff down in the
6500                            // command-serving branch below rather than sleeping
6501                            // here: commands cannot be received from inside this
6502                            // select arm, and an operator disable or drain that
6503                            // arrives during the backoff must cancel the pending
6504                            // respawn instead of waiting for it to spawn first.
6505                            child = None;
6506                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6507                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6508                        }
6509                    }
6510                }
6511                command = commands.recv() => {
6512                    #[cfg(test)]
6513                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6514                    let Some(command) = command else {
6515                        return;
6516                    };
6517                    if !handle_supervisor_command(
6518                        command,
6519                        &mut spec,
6520                        &mut runtime,
6521                        &registry,
6522                        &process_liveness,
6523                        &snapshot,
6524                        &mut child,
6525                        &mut commands,
6526                        &mut requeued,
6527                    ).await {
6528                        return;
6529                    }
6530                }
6531                _ = async {
6532                    match registration_changes.as_mut() {
6533                        Some(changes) => { let _ = changes.changed().await; }
6534                        None => std::future::pending().await,
6535                    }
6536                }, if wire_child => {
6537                    #[cfg(test)]
6538                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6539                }
6540                _ = &mut probe_sleep => {
6541                    #[cfg(test)]
6542                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6543                    if health_probe.due() {
6544                        run_health_probe_cycle(
6545                            &spec,
6546                            &runtime,
6547                            &registry,
6548                            &process_liveness,
6549                            &snapshot,
6550                            &mut child,
6551                        ).await;
6552                        if child.is_none() {
6553                            runtime.start_requester = None;
6554                            runtime.pending_activity_restart = false;
6555                        }
6556                        if child.is_some() {
6557                            health_probe.schedule_next(&spec, runtime.health.cadence);
6558                        }
6559                    }
6560                }
6561            }
6562        } else if let Some(pending) = pending_respawn {
6563            tokio::select! {
6564                _ = sleep_until(pending.deadline) => {
6565                    pending_respawn = None;
6566                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6567                    // A command handled below while the backoff elapsed may
6568                    // have stopped the module; never respawn past an operator's
6569                    // disable or drain.
6570                    if !respawn_still_pending(&snapshot) {
6571                        continue;
6572                    }
6573                    // The daemon began shutting down during the backoff: the
6574                    // spawn would be refused anyway, and refusing it here
6575                    // leaves the module stopped instead of reporting a
6576                    // failed restart.
6577                    if runtime.child_roster.is_closed() {
6578                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6579                            state.state = ModuleState::Stopped;
6580                        });
6581                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6582                        continue;
6583                    }
6584                    if let Err(err) = release_dead_registration(
6585                        &registry,
6586                        runtime.forwarding.as_deref(),
6587                        &snapshot,
6588                        &spec.module_id,
6589                    ).await {
6590                        fail_snapshot(&snapshot, Some(&spec.module_id), None);
6591                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6592                        continue;
6593                    }
6594
6595                    if matches!(pending.kind, RespawnKind::Reload) {
6596                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6597                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
6598                        if result.is_ok() && runtime.pending_activity_restart {
6599                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6600                        }
6601                        runtime.pending_activity_restart = false;
6602                        if let Some(reply) = reply { let _ = reply.send(result); }
6603                        continue;
6604                    }
6605                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6606                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6607                        Ok(next_child) => {
6608                            child = Some(next_child);
6609                            if runtime.pending_activity_restart {
6610                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6611                                runtime.pending_activity_restart = false;
6612                            }
6613                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6614                        }
6615                        Err(err) => {
6616                            fail_snapshot(&snapshot, Some(&spec.module_id), None);
6617                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6618                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6619                        }
6620                    }
6621                }
6622                command = commands.recv() => {
6623                    let Some(command) = command else {
6624                        return;
6625                    };
6626                    if !handle_supervisor_command(
6627                        command,
6628                        &mut spec,
6629                        &mut runtime,
6630                        &registry,
6631                        &process_liveness,
6632                        &snapshot,
6633                        &mut child,
6634                        &mut commands,
6635                        &mut requeued,
6636                    ).await {
6637                        return;
6638                    }
6639                    // Reconcile the pending respawn with what the command did:
6640                    // a start may already have spawned a fresh child,
6641                    // while a disable or drain moved the snapshot out of the
6642                    // state the respawn was counting down from.
6643                    if child.is_some() || !respawn_still_pending(&snapshot) {
6644                        pending_respawn = None;
6645                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6646                    }
6647                }
6648            }
6649        } else {
6650            let command = if lock_snapshot(&snapshot)
6651                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
6652            {
6653                tokio::select! {
6654                    command = commands.recv() => command,
6655                    _ = previous_instance_recheck.tick() => {
6656                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
6657                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
6658                            if enabled && !runtime.child_roster.is_closed() {
6659                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6660                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6661                                    Ok(next_child) => child = Some(next_child),
6662                                    Err(error) => {
6663                                        fail_snapshot(&snapshot, Some(&spec.module_id), None);
6664                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
6665                                    }
6666                                }
6667                            } else {
6668                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6669                                    state.waiting_for_previous_pid = None;
6670                                    state.waiting_for_previous_detail = None;
6671                                });
6672                            }
6673                        }
6674                        continue;
6675                    }
6676                }
6677            } else {
6678                commands.recv().await
6679            };
6680            let Some(command) = command else {
6681                return;
6682            };
6683            if !handle_supervisor_command(
6684                command,
6685                &mut spec,
6686                &mut runtime,
6687                &registry,
6688                &process_liveness,
6689                &snapshot,
6690                &mut child,
6691                &mut commands,
6692                &mut requeued,
6693            )
6694            .await
6695            {
6696                return;
6697            }
6698        }
6699    }
6700}
6701
6702fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
6703    info!(
6704        module_id,
6705        restart_in_window = schedule.restart_in_window,
6706        delay_ms = schedule.delay.as_millis() as u64,
6707        "respawning after crash"
6708    );
6709}
6710
6711/// Whether the respawn a backoff was counting down to is still wanted. A
6712/// disable or drain handled while the backoff elapsed moves the snapshot out
6713/// of `Restarting`, and the operator's stop must win over the pending respawn,
6714/// so every sleep-then-spawn path re-validates against the live snapshot
6715/// instead of assuming the state it left behind still holds.
6716fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
6717    matches!(
6718        lock_snapshot(snapshot),
6719        Ok(state) if state.enabled && state.state == ModuleState::Restarting
6720    )
6721}
6722
6723enum NextAction {
6724    Stop {
6725        registration_released: bool,
6726    },
6727    Restart {
6728        schedule: Option<CrashRestartSchedule>,
6729    },
6730}
6731
6732#[allow(clippy::too_many_arguments)]
6733async fn handle_supervisor_command(
6734    command: SupervisorCommand,
6735    spec: &mut ModuleSpec,
6736    runtime: &mut SupervisorRuntimeConfig,
6737    registry: &Arc<Registry>,
6738    process_liveness: &SupervisorProcessLiveness,
6739    snapshot: &SharedSnapshot,
6740    child: &mut Option<SupervisedChild>,
6741    commands: &mut mpsc::Receiver<SupervisorCommand>,
6742    requeued: &mut VecDeque<SupervisorCommand>,
6743) -> bool {
6744    let command =
6745        if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
6746            match command {
6747                SupervisorCommand::Restart { reply, .. }
6748                | SupervisorCommand::Reload { reply, .. }
6749                | SupervisorCommand::Swap { reply, .. } => {
6750                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6751                    return true;
6752                }
6753                SupervisorCommand::SetEnabled {
6754                    enabled: true,
6755                    reply,
6756                    ..
6757                } => {
6758                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6759                    return true;
6760                }
6761                other => other,
6762            }
6763        } else {
6764            command
6765        };
6766    match command {
6767        SupervisorCommand::Drain { requester, reply } => {
6768            // A plain stop runs no forwarding drain, so nothing reaches the
6769            // module over its connection before the wait: ask by signal.
6770            let result = drain_optional_child(
6771                &spec.module_id,
6772                spec.protocol,
6773                StopNotice::NotSent,
6774                registry,
6775                runtime.forwarding.as_deref(),
6776                snapshot,
6777                &runtime.terminal_ring,
6778                &runtime.spawn_events,
6779                child,
6780                runtime.drain_timeout,
6781                ModuleState::Stopped,
6782                None,
6783            )
6784            .await;
6785            let registration_released = result.is_ok();
6786            if registration_released {
6787                runtime.record_activity_transition(
6788                    "subc.module_stopped",
6789                    &spec.module_id,
6790                    requester,
6791                );
6792            }
6793            let _ = reply.send(result);
6794            if registration_released {
6795                process_liveness.untrack_if_current(&spec.module_id, snapshot);
6796            }
6797            false
6798        }
6799        SupervisorCommand::Retire { requester, reply } => {
6800            let result = async {
6801                let stop_notice = begin_forwarding_drain_if_configured(
6802                    spec,
6803                    runtime,
6804                    registry,
6805                    snapshot,
6806                    None,
6807                    RouteCloseReason::Disable,
6808                )
6809                .await?;
6810                drain_optional_child(
6811                    &spec.module_id,
6812                    spec.protocol,
6813                    stop_notice,
6814                    registry,
6815                    runtime.forwarding.as_deref(),
6816                    snapshot,
6817                    &runtime.terminal_ring,
6818                    &runtime.spawn_events,
6819                    child,
6820                    runtime.drain_timeout,
6821                    ModuleState::Stopped,
6822                    None,
6823                )
6824                .await
6825            }
6826            .await;
6827            let registration_released = result.is_ok();
6828            if registration_released {
6829                runtime.record_activity_transition(
6830                    "subc.module_stopped",
6831                    &spec.module_id,
6832                    requester,
6833                );
6834            }
6835            let _ = reply.send(result);
6836            if registration_released {
6837                process_liveness.untrack_if_current(&spec.module_id, snapshot);
6838            }
6839            false
6840        }
6841        SupervisorCommand::Restart {
6842            requester,
6843            drain_timeout_ms,
6844            received_at_generation,
6845            queued_at,
6846            reply,
6847        } => {
6848            // Without this line a restart that waited in the queue (behind a
6849            // health probe cycle or another command) was invisible: the log
6850            // showed only the drain timing out, minutes after the operator's call.
6851            info!(
6852                module_id = %spec.module_id,
6853                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
6854                "restart command dequeued"
6855            );
6856            // ACK AT INITIATION, not completion. The blocking form deadlocked any
6857            // caller whose own request lane rides the module being restarted: the
6858            // caller's in-flight request keeps the drain from quiescing, the drain
6859            // keeps the restart from completing, and the completion keeps the reply
6860            // from releasing the caller — so the drain always timed out and cut the
6861            // initiator with a GOODBYE, even on a healthy module. Replying once the
6862            // restart is validated lets a self-lane caller settle, which is exactly
6863            // what makes the drain succeed. Completion is observable via
6864            // supervisor.list / module status; a post-ack failure lands the module
6865            // in a visible terminal state below rather than in a reply nobody can
6866            // receive.
6867            let validation = match lock_snapshot(snapshot) {
6868                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
6869                    module_id: spec.module_id.clone(),
6870                }),
6871                Ok(_) => Ok(()),
6872                Err(err) => Err(err),
6873            };
6874            let initiated = validation.is_ok();
6875            let _ = reply.send(validation);
6876            // A restart asks for a fresh process. Commands run one at a time,
6877            // so a restart queued behind another restart (two operator calls
6878            // in quick succession) is dequeued the moment the first one has
6879            // spawned its replacement -- before that process has sent HELLO.
6880            // Running it would drain and kill the process the first restart
6881            // just produced, which is the opposite of what both callers asked
6882            // for. If a process spawned after this request was received is
6883            // still supervised, the request is already satisfied. Not when the
6884            // configuration changed since that spawn: then the newer process
6885            // predates the spec this restart may exist to apply.
6886            let satisfied_by_generation = if initiated && child.is_some() {
6887                lock_snapshot(snapshot).ok().and_then(|state| {
6888                    (state.spawn_generation > received_at_generation
6889                        && !state.configuration_updated_since_spawn)
6890                        .then_some(state.spawn_generation)
6891                })
6892            } else {
6893                None
6894            };
6895            let satisfied_by_pending = initiated
6896                && child.is_none()
6897                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
6898                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
6899                    if pending {
6900                        state.coalesced_restart_pending = true;
6901                    }
6902                    pending
6903                });
6904            if satisfied_by_pending {
6905                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
6906            } else if let Some(generation) = satisfied_by_generation {
6907                info!(
6908                    module_id = %spec.module_id,
6909                    received_at_generation,
6910                    "restart already satisfied by generation {generation}; not restarting again"
6911                );
6912            } else if initiated {
6913                // Precedence: this restart's operator override, else the module's
6914                // configured budget (already resolved into the runtime).
6915                let drain_timeout = drain_timeout_ms
6916                    .map(Duration::from_millis)
6917                    .unwrap_or(runtime.drain_timeout);
6918                if let Err(err) = restart_child(
6919                    spec,
6920                    runtime,
6921                    registry,
6922                    process_liveness,
6923                    snapshot,
6924                    child,
6925                    drain_timeout,
6926                )
6927                .await
6928                {
6929                    warn!(
6930                        module_id = %spec.module_id,
6931                        error = %err,
6932                        "operator restart failed after initiation ack; module state carries the outcome"
6933                    );
6934                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6935                        state.state = ModuleState::Failed;
6936                        clear_current_process_facts(state);
6937                    });
6938                } else {
6939                    runtime.start_requester = requester;
6940                    runtime.pending_activity_restart = true;
6941                }
6942            }
6943            true
6944        }
6945        SupervisorCommand::Reload { requester, reply } => {
6946            let result =
6947                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
6948            if result.is_ok() {
6949                runtime.start_requester = requester;
6950                runtime.pending_activity_restart = true;
6951            }
6952            if result.is_ok()
6953                && runtime
6954                    .scheduled_respawn
6955                    .lock()
6956                    .unwrap_or_else(|p| p.into_inner())
6957                    .as_ref()
6958                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
6959            {
6960                *runtime
6961                    .deferred_reload_reply
6962                    .lock()
6963                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
6964            } else {
6965                let _ = reply.send(result);
6966            }
6967            true
6968        }
6969        SupervisorCommand::SetEnabled {
6970            enabled,
6971            requester,
6972            reply,
6973        } => {
6974            let previous_requester = runtime.start_requester.clone();
6975            if enabled {
6976                runtime.start_requester = requester.clone();
6977            }
6978            let result = set_child_enabled(
6979                spec,
6980                runtime,
6981                registry,
6982                process_liveness,
6983                snapshot,
6984                child,
6985                enabled,
6986            )
6987            .await;
6988            if matches!(result, Ok(true)) {
6989                runtime.pending_activity_restart = false;
6990                if enabled {
6991                    runtime.start_requester = requester;
6992                } else {
6993                    runtime.start_requester = None;
6994                    runtime.record_activity_transition(
6995                        "subc.module_disabled",
6996                        &spec.module_id,
6997                        requester,
6998                    );
6999                }
7000            } else {
7001                runtime.start_requester = previous_requester;
7002            }
7003            let _ = reply.send(result);
7004            true
7005        }
7006        SupervisorCommand::UpdateConfiguration {
7007            spec: next_spec,
7008            health,
7009            drain_timeout_ms,
7010            reply,
7011        } => {
7012            if let Some(handle) = &runtime.supervisor_handle {
7013                handle.apply_identity_configuration(&next_spec);
7014            }
7015            *spec = next_spec;
7016            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7017                state.configuration_updated_since_spawn = true;
7018            });
7019            let health_changed = runtime.health != health;
7020            runtime.health = health;
7021            // Reset the cadence and old endpoint's failure streak on a live
7022            // health-policy change rather than waiting for its old deadline.
7023            if health_changed {
7024                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7025                    state.health = ModuleHealthStatus::default();
7026                });
7027            }
7028            runtime.drain_timeout = drain_timeout_ms
7029                .map(Duration::from_millis)
7030                .unwrap_or(runtime.default_drain_timeout);
7031            *runtime
7032                .effective_drain_timeout
7033                .lock()
7034                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7035            let _ = reply.send(());
7036            true
7037        }
7038        SupervisorCommand::Swap {
7039            requester,
7040            ready_timeout,
7041            reply,
7042        } => {
7043            let mut swap_runtime = runtime.clone();
7044            swap_runtime.start_requester = requester;
7045            let before = lock_snapshot(snapshot)
7046                .ok()
7047                .map(|state| state.spawn_generation);
7048            let end = swap::run_swap(
7049                spec,
7050                &swap_runtime,
7051                registry,
7052                process_liveness,
7053                snapshot,
7054                child,
7055                commands,
7056                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7057                reply,
7058            )
7059            .await;
7060            if before
7061                != lock_snapshot(snapshot)
7062                    .ok()
7063                    .map(|state| state.spawn_generation)
7064            {
7065                runtime.start_requester = swap_runtime.start_requester;
7066            }
7067            requeued.extend(end.requeue);
7068            true
7069        }
7070    }
7071}
7072
7073async fn restart_child(
7074    spec: &ModuleSpec,
7075    runtime: &SupervisorRuntimeConfig,
7076    registry: &Registry,
7077    process_liveness: &SupervisorProcessLiveness,
7078    snapshot: &SharedSnapshot,
7079    child: &mut Option<SupervisedChild>,
7080    drain_timeout: Duration,
7081) -> Result<(), SuperviseError> {
7082    // Restart cycles a running module; it must not silently start a disabled one.
7083    if !lock_snapshot(snapshot)?.enabled {
7084        return Err(SuperviseError::Disabled {
7085            module_id: spec.module_id.clone(),
7086        });
7087    }
7088    let stop_notice = begin_forwarding_drain_with_timeout(
7089        spec,
7090        runtime,
7091        registry,
7092        snapshot,
7093        None,
7094        RouteCloseReason::Restart,
7095        drain_timeout,
7096    )
7097    .await?;
7098
7099    if child.is_some() {
7100        drain_optional_child(
7101            &spec.module_id,
7102            spec.protocol,
7103            stop_notice,
7104            registry,
7105            runtime.forwarding.as_deref(),
7106            snapshot,
7107            &runtime.terminal_ring,
7108            &runtime.spawn_events,
7109            child,
7110            drain_timeout,
7111            ModuleState::Restarting,
7112            Some(true),
7113        )
7114        .await?;
7115    } else {
7116        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7117            state.enabled = true;
7118            state.state = ModuleState::Restarting;
7119            clear_current_process_facts(state);
7120        })?;
7121        release_dead_registration(
7122            registry,
7123            runtime.forwarding.as_deref(),
7124            snapshot,
7125            &spec.module_id,
7126        )
7127        .await?;
7128    }
7129
7130    reset_restart_count(snapshot, &spec.module_id)?;
7131    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7132    schedule_respawn(
7133        runtime,
7134        snapshot,
7135        &spec.module_id,
7136        runtime.restart_policy.backoff,
7137        RespawnKind::Spawn,
7138    )
7139}
7140
7141async fn reload_child(
7142    spec: &ModuleSpec,
7143    runtime: &SupervisorRuntimeConfig,
7144    registry: &Registry,
7145    process_liveness: &SupervisorProcessLiveness,
7146    snapshot: &SharedSnapshot,
7147    child: &mut Option<SupervisedChild>,
7148) -> Result<(), SuperviseError> {
7149    // Reload cycles a running module; it must not silently start a disabled one.
7150    if !lock_snapshot(snapshot)?.enabled {
7151        return Err(SuperviseError::Disabled {
7152            module_id: spec.module_id.clone(),
7153        });
7154    }
7155    let stop_notice = begin_forwarding_drain(
7156        spec,
7157        runtime,
7158        registry,
7159        snapshot,
7160        Some(true),
7161        RouteCloseReason::Reload,
7162    )
7163    .await?;
7164
7165    if child.is_some() {
7166        drain_optional_child(
7167            &spec.module_id,
7168            spec.protocol,
7169            stop_notice,
7170            registry,
7171            runtime.forwarding.as_deref(),
7172            snapshot,
7173            &runtime.terminal_ring,
7174            &runtime.spawn_events,
7175            child,
7176            runtime.drain_timeout,
7177            ModuleState::Restarting,
7178            Some(true),
7179        )
7180        .await?;
7181    } else {
7182        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7183            state.enabled = true;
7184            state.state = ModuleState::Restarting;
7185            clear_current_process_facts(state);
7186        })?;
7187        release_dead_registration(
7188            registry,
7189            runtime.forwarding.as_deref(),
7190            snapshot,
7191            &spec.module_id,
7192        )
7193        .await?;
7194    }
7195
7196    reset_restart_count(snapshot, &spec.module_id)?;
7197    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7198    schedule_respawn(
7199        runtime,
7200        snapshot,
7201        &spec.module_id,
7202        runtime.restart_policy.backoff,
7203        RespawnKind::Reload,
7204    )
7205}
7206
7207async fn finish_reload_child(
7208    spec: &ModuleSpec,
7209    runtime: &SupervisorRuntimeConfig,
7210    registry: &Registry,
7211    process_liveness: &SupervisorProcessLiveness,
7212    snapshot: &SharedSnapshot,
7213    child: &mut Option<SupervisedChild>,
7214) -> Result<(), SuperviseError> {
7215    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7216    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7217        Ok(next_child) => next_child,
7218        Err(err) => {
7219            return handle_reload_spawn_failure(
7220                spec,
7221                runtime,
7222                process_liveness,
7223                snapshot,
7224                child,
7225                format!("new child failed to spawn: {err}"),
7226            )
7227            .await;
7228        }
7229    };
7230    *child = Some(next_child);
7231
7232    let wait_outcome = {
7233        let active_child = child.as_mut().expect("new reload child was just stored");
7234        wait_for_registration_after_reload(
7235            registry,
7236            &spec.module_id,
7237            snapshot,
7238            active_child,
7239            REGISTRY_RELEASE_TIMEOUT,
7240        )
7241        .await?
7242    };
7243
7244    match wait_outcome {
7245        RegistrationWaitOutcome::Registered => {
7246            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7247            Ok(())
7248        }
7249        RegistrationWaitOutcome::Exited(exit_report) => {
7250            if let Some(active_child) = child.as_mut() {
7251                active_child.drain_stderr(&spec.module_id).await;
7252            }
7253            // Keep the reaped child's roster guard until its terminal is written.
7254            // Shutdown waits on that guard, not on the child Option used for respawn.
7255            let mut exited_child = child.take().expect("exited reload child is still stored");
7256            #[cfg(test)]
7257            if let Some(gate) = &runtime.test_reload_exit_record_gate {
7258                gate.reached.notify_one();
7259                gate.resume.notified().await;
7260            }
7261            let result = handle_reload_child_registration_failure(
7262                spec,
7263                runtime,
7264                registry,
7265                process_liveness,
7266                snapshot,
7267                child,
7268                ReloadRegistrationFailure {
7269                    exit_report: registration_failure_exit_report(exit_report),
7270                    reason: exited_child
7271                        .spawn_failure
7272                        .clone()
7273                        .unwrap_or_else(|| "new child exited before registering".to_string()),
7274                },
7275            )
7276            .await;
7277            exited_child.release_roster();
7278            result
7279        }
7280        RegistrationWaitOutcome::TimedOut => {
7281            let mut timed_out_child = child
7282                .take()
7283                .expect("timed-out reload child is still running");
7284            timed_out_child
7285                .start_kill()
7286                .map_err(|source| SuperviseError::Kill {
7287                    module_id: spec.module_id.clone(),
7288                    source,
7289                })?;
7290            let status = timed_out_child
7291                .wait()
7292                .await
7293                .map_err(|source| SuperviseError::Wait {
7294                    module_id: spec.module_id.clone(),
7295                    source,
7296                })?;
7297            timed_out_child.drain_stderr(&spec.module_id).await;
7298            handle_reload_child_registration_failure(
7299                spec,
7300                runtime,
7301                registry,
7302                process_liveness,
7303                snapshot,
7304                child,
7305                ReloadRegistrationFailure {
7306                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7307                        snapshot,
7308                        &timed_out_child,
7309                        &status,
7310                    )),
7311                    reason: format!(
7312                        "new child did not register within {:?}",
7313                        REGISTRY_RELEASE_TIMEOUT
7314                    ),
7315                },
7316            )
7317            .await
7318        }
7319    }
7320}
7321
7322async fn set_child_enabled(
7323    spec: &ModuleSpec,
7324    runtime: &SupervisorRuntimeConfig,
7325    registry: &Registry,
7326    process_liveness: &SupervisorProcessLiveness,
7327    snapshot: &SharedSnapshot,
7328    child: &mut Option<SupervisedChild>,
7329    enabled: bool,
7330) -> Result<bool, SuperviseError> {
7331    let (current_enabled, current_state, respawn_pending) = {
7332        let state = lock_snapshot(snapshot)?;
7333        (state.enabled, state.state, state.respawn_pending)
7334    };
7335    // `start` (enable on an already-enabled module) heals TERMINAL states instead
7336    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
7337    // clean (Stopped) has no live process and no other in-band recovery — the
7338    // operator's start is the explicit recovery act and resets the budget. Without
7339    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
7340    // which the 2026-07-14 aft outage proved is a trap when the failed module is
7341    // the one providing every agent's shell.
7342    let revive_terminal = enabled
7343        && current_enabled
7344        && child.is_none()
7345        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7346            || (current_state == ModuleState::Restarting && !respawn_pending));
7347    if current_enabled == enabled && !revive_terminal {
7348        return Ok(false);
7349    }
7350
7351    if enabled {
7352        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7353            state.enabled = true;
7354            state.state = ModuleState::Starting;
7355            clear_current_process_facts(state);
7356        })?;
7357        #[cfg(test)]
7358        if runtime.test_seed_stale_facts_before_enable_spawn {
7359            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7360                state.process_alive = true;
7361                state.pid = Some(41);
7362                state.spawned_at_ms = Some(42);
7363                state.spawned_from = Some(PathBuf::from("/spawned/module"));
7364                state.spawned_file_identity = Some(SpawnedFileIdentity {
7365                    device: 43,
7366                    inode: 44,
7367                    #[cfg(windows)]
7368                    windows_image: None,
7369                });
7370            })?;
7371        }
7372        release_dead_registration(
7373            registry,
7374            runtime.forwarding.as_deref(),
7375            snapshot,
7376            &spec.module_id,
7377        )
7378        .await?;
7379        reset_restart_count(snapshot, &spec.module_id)?;
7380        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7381        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7382            Ok(next_child) => next_child,
7383            Err(err) => {
7384                if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7385                    state.state = ModuleState::Failed;
7386                    clear_current_process_facts(state);
7387                }) {
7388                    error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7389                }
7390                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7391                return Err(err);
7392            }
7393        };
7394        *child = Some(next_child);
7395        debug!(module_id = %spec.module_id, "supervised module enabled");
7396        Ok(true)
7397    } else {
7398        let stop_notice = begin_forwarding_drain_if_configured(
7399            spec,
7400            runtime,
7401            registry,
7402            snapshot,
7403            Some(false),
7404            RouteCloseReason::Disable,
7405        )
7406        .await?;
7407        drain_optional_child(
7408            &spec.module_id,
7409            spec.protocol,
7410            stop_notice,
7411            registry,
7412            runtime.forwarding.as_deref(),
7413            snapshot,
7414            &runtime.terminal_ring,
7415            &runtime.spawn_events,
7416            child,
7417            runtime.drain_timeout,
7418            ModuleState::Disabled,
7419            Some(false),
7420        )
7421        .await?;
7422        debug!(module_id = %spec.module_id, "supervised module disabled");
7423        Ok(true)
7424    }
7425}
7426
7427#[allow(clippy::too_many_arguments)]
7428async fn on_child_exit(
7429    spec: &ModuleSpec,
7430    policy: RestartPolicy,
7431    registry: &Registry,
7432    snapshot: &SharedSnapshot,
7433    terminal_ring: &Arc<Mutex<TerminalRing>>,
7434    spawn_events: &SpawnEventFeed,
7435    roster: &ChildRoster,
7436    exit_report: ExitReport,
7437) -> NextAction {
7438    // Once the daemon has begun shutting down, no exit is a crash to recover
7439    // from: the module is exiting because the daemon is going away (EOF on its
7440    // connection, or a service manager signalling the whole cgroup). Record it
7441    // as such and never schedule a respawn, which would only start a process
7442    // for the shutdown to end again.
7443    if roster.is_closed() {
7444        return on_child_exit_during_daemon_shutdown(
7445            spec,
7446            registry,
7447            snapshot,
7448            terminal_ring,
7449            spawn_events,
7450            exit_report,
7451        )
7452        .await;
7453    }
7454    // Every stop the supervisor itself asks for (operator stop, disable,
7455    // restart, reload, swap, a health restart, a drain that runs out of budget)
7456    // takes the child out of the supervise loop and reaps it in
7457    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
7458    // that reaches this point was not requested by the daemon.
7459    //
7460    // For a subc-wire module a clean exit is still a stop: those modules are
7461    // written to re-raise SIGTERM, so a stray outside signal already reads as a
7462    // crash, and exiting 0 is a deliberate choice the module made. A
7463    // `protocol: "none"` module is a stock program we cannot change, and many
7464    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
7465    // stop would leave the module down for good after any stray signal, so it
7466    // goes through the crash path instead: it spends restart budget, respawns
7467    // with the crash backoff, and ends `failed` when the budget runs out.
7468    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7469        && running_protocol(spec, snapshot) == ModuleProtocol::None;
7470    match exit_report.kind {
7471        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7472            info!(
7473                module_id = %spec.module_id,
7474                exit_code = ?exit_report.code,
7475                exit_signal = ?exit_report.signal,
7476                "supervised module exited cleanly"
7477            );
7478            if let Err(err) = publish_after_terminal_record(
7479                &spec.module_id,
7480                || {
7481                    record_terminal(
7482                        &spec.module_id,
7483                        terminal_ring,
7484                        spawn_events,
7485                        &exit_report,
7486                        TerminalDisposition::Stopped,
7487                    )
7488                },
7489                || {
7490                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7491                        state.state = ModuleState::Stopped;
7492                        clear_current_process_facts(state);
7493                        state.last_exit = Some(exit_report.clone());
7494                    })
7495                },
7496            ) {
7497                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7498            }
7499            let registration_released = match wait_for_registration_release(
7500                registry,
7501                &spec.module_id,
7502                REGISTRY_RELEASE_TIMEOUT,
7503            )
7504            .await
7505            {
7506                Ok(()) => true,
7507                Err(err) => {
7508                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7509                    false
7510                }
7511            };
7512            NextAction::Stop {
7513                registration_released,
7514            }
7515        }
7516        ExitKind::Clean | ExitKind::Crash => {
7517            if unrequested_clean_exit_of_protocol_none {
7518                warn!(
7519                    module_id = %spec.module_id,
7520                    exit_code = ?exit_report.code,
7521                    exit_signal = ?exit_report.signal,
7522                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
7523                );
7524            } else {
7525                warn!(
7526                    module_id = %spec.module_id,
7527                    exit_code = ?exit_report.code,
7528                    exit_signal = ?exit_report.signal,
7529                    "supervised module exited abnormally (crash)"
7530                );
7531            }
7532            let mut restart_schedule = None;
7533            let mut disposition = TerminalDisposition::Disabled;
7534            let mut final_state = ModuleState::Disabled;
7535            // Set only when the budget is what stopped the module, so the
7536            // terminal record says which limit was hit rather than leaving
7537            // `failed` to be read as "crashed once, badly".
7538            let mut disposition_detail = lock_snapshot(snapshot)
7539                .ok()
7540                .and_then(|mut state| state.spawn_failure.take());
7541            let now = Instant::now();
7542            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7543                if state.enabled {
7544                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
7545                        // Reserving the last slot exhausts the budget for the
7546                        // replacement, not for this exit. Publish its scheduled
7547                        // recovery atomically with the reservation so connection
7548                        // cleanup sees it while terminal recording still holds
7549                        // the visible state at Running.
7550                        state.respawn_pending = true;
7551                        final_state = ModuleState::Restarting;
7552                        restart_schedule = Some(schedule);
7553                        disposition = TerminalDisposition::Restarting;
7554                    } else {
7555                        final_state = ModuleState::Failed;
7556                        disposition = TerminalDisposition::Failed;
7557                        let budget = policy.budget_exhausted_detail();
7558                        disposition_detail =
7559                            Some(disposition_detail.take().map_or_else(
7560                                || budget.clone(),
7561                                |cause| format!("{cause}; {budget}"),
7562                            ));
7563                    }
7564                } else {
7565                    disposition = TerminalDisposition::Disabled;
7566                }
7567            }) {
7568                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7569                return NextAction::Stop {
7570                    registration_released: false,
7571                };
7572            }
7573            if disposition == TerminalDisposition::Failed {
7574                // The window is in the message, not only in the fields: this line
7575                // is read in a scrollback where a bare `max_restarts=3` reads as a
7576                // lifetime cap and sends the operator looking for three crashes
7577                // that never happened together.
7578                error!(
7579                    module_id = %spec.module_id,
7580                    max_restarts = policy.max_restarts,
7581                    window_secs = policy.window.as_secs(),
7582                    "module stopped: {}",
7583                    policy.budget_exhausted_detail()
7584                );
7585            }
7586            if let Err(err) = publish_after_terminal_record(
7587                &spec.module_id,
7588                || {
7589                    record_terminal_with_detail(
7590                        &spec.module_id,
7591                        terminal_ring,
7592                        spawn_events,
7593                        &exit_report,
7594                        disposition,
7595                        disposition_detail,
7596                    )
7597                },
7598                || {
7599                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7600                        state.state = final_state;
7601                        clear_current_process_facts(state);
7602                        state.last_exit = Some(exit_report.clone());
7603                    })
7604                },
7605            ) {
7606                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7607            }
7608
7609            if let Some(schedule) = restart_schedule {
7610                NextAction::Restart {
7611                    schedule: Some(schedule),
7612                }
7613            } else {
7614                let registration_released = match wait_for_registration_release(
7615                    registry,
7616                    &spec.module_id,
7617                    REGISTRY_RELEASE_TIMEOUT,
7618                )
7619                .await
7620                {
7621                    Ok(()) => true,
7622                    Err(err) => {
7623                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7624                        false
7625                    }
7626                };
7627                NextAction::Stop {
7628                    registration_released,
7629                }
7630            }
7631        }
7632        ExitKind::DeliberateSeverance => {
7633            warn!(
7634                module_id = %spec.module_id,
7635                exit_code = ?exit_report.code,
7636                exit_signal = ?exit_report.signal,
7637                "supervised module exited after deliberate connection severance"
7638            );
7639            let should_restart = match lock_snapshot(snapshot) {
7640                Ok(state) => state.enabled,
7641                Err(err) => {
7642                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
7643                    return NextAction::Stop {
7644                        registration_released: false,
7645                    };
7646                }
7647            };
7648            let final_state = if should_restart {
7649                ModuleState::Restarting
7650            } else {
7651                ModuleState::Disabled
7652            };
7653            if let Err(err) = publish_after_terminal_record(
7654                &spec.module_id,
7655                || {
7656                    record_terminal(
7657                        &spec.module_id,
7658                        terminal_ring,
7659                        spawn_events,
7660                        &exit_report,
7661                        terminal_disposition(final_state),
7662                    )
7663                },
7664                || {
7665                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7666                        state.state = final_state;
7667                        clear_current_process_facts(state);
7668                        state.last_exit = Some(exit_report.clone());
7669                        state.lifetime_restarts += 1;
7670                    })
7671                },
7672            ) {
7673                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
7674                return NextAction::Stop {
7675                    registration_released: false,
7676                };
7677            }
7678
7679            if should_restart {
7680                NextAction::Restart { schedule: None }
7681            } else {
7682                let registration_released = match wait_for_registration_release(
7683                    registry,
7684                    &spec.module_id,
7685                    REGISTRY_RELEASE_TIMEOUT,
7686                )
7687                .await
7688                {
7689                    Ok(()) => true,
7690                    Err(err) => {
7691                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
7692                        false
7693                    }
7694                };
7695                NextAction::Stop {
7696                    registration_released,
7697                }
7698            }
7699        }
7700    }
7701}
7702
7703async fn on_child_exit_during_daemon_shutdown(
7704    spec: &ModuleSpec,
7705    registry: &Registry,
7706    snapshot: &SharedSnapshot,
7707    terminal_ring: &Arc<Mutex<TerminalRing>>,
7708    spawn_events: &SpawnEventFeed,
7709    exit_report: ExitReport,
7710) -> NextAction {
7711    info!(
7712        module_id = %spec.module_id,
7713        exit_code = ?exit_report.code,
7714        exit_signal = ?exit_report.signal,
7715        exit_kind = ?exit_report.kind,
7716        "supervised module exited during daemon shutdown; not restarting it"
7717    );
7718    if let Err(err) = publish_after_terminal_record(
7719        &spec.module_id,
7720        || {
7721            record_terminal(
7722                &spec.module_id,
7723                terminal_ring,
7724                spawn_events,
7725                &exit_report,
7726                TerminalDisposition::DaemonShutdown,
7727            )
7728        },
7729        || {
7730            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7731                state.state = ModuleState::Stopped;
7732                clear_current_process_facts(state);
7733                state.last_exit = Some(exit_report.clone());
7734            })
7735        },
7736    ) {
7737        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
7738    }
7739    let registration_released =
7740        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
7741            .await
7742            .is_ok();
7743    NextAction::Stop {
7744        registration_released,
7745    }
7746}
7747
7748// Keep the status lock out of terminal I/O. Readers may observe the previous
7749// state while recording is in progress, but never a completed exit without its
7750// history. Journal errors are logged by the journal and retained in the ring;
7751// a recording panic still publishes the final state before resuming the unwind.
7752fn publish_after_terminal_record<T>(
7753    module_id: &str,
7754    record: impl FnOnce(),
7755    publish: impl FnOnce() -> T,
7756) -> T {
7757    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
7758    let published = publish();
7759    if let Err(panic) = recorded {
7760        error!(
7761            module_id,
7762            "terminal recording panicked; published final state before resuming unwind"
7763        );
7764        std::panic::resume_unwind(panic);
7765    }
7766    published
7767}
7768
7769fn fail_after_wait_error(
7770    module_id: &str,
7771    snapshot: &SharedSnapshot,
7772    terminal_ring: &Arc<Mutex<TerminalRing>>,
7773    spawn_events: &SpawnEventFeed,
7774    detail: String,
7775) {
7776    let report = wait_error_exit_report();
7777    publish_after_terminal_record(
7778        module_id,
7779        || {
7780            record_terminal_with_detail(
7781                module_id,
7782                terminal_ring,
7783                spawn_events,
7784                &report,
7785                TerminalDisposition::Failed,
7786                Some(detail),
7787            )
7788        },
7789        || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
7790    );
7791}
7792
7793#[cfg(test)]
7794fn record_wait_error_terminal(
7795    module_id: &str,
7796    terminal_ring: &Arc<Mutex<TerminalRing>>,
7797    spawn_events: &SpawnEventFeed,
7798) {
7799    record_terminal(
7800        module_id,
7801        terminal_ring,
7802        spawn_events,
7803        &wait_error_exit_report(),
7804        TerminalDisposition::Failed,
7805    );
7806}
7807
7808fn record_terminal(
7809    module_id: &str,
7810    terminal_ring: &Arc<Mutex<TerminalRing>>,
7811    spawn_events: &SpawnEventFeed,
7812    exit_report: &ExitReport,
7813    disposition: TerminalDisposition,
7814) {
7815    record_terminal_with_detail(
7816        module_id,
7817        terminal_ring,
7818        spawn_events,
7819        exit_report,
7820        disposition,
7821        None,
7822    );
7823}
7824
7825/// The ring lock is held only to capture the read (see
7826/// `TerminalJournal::capture_read`), so this module's exits keep recording
7827/// while the journal files are read. Blocking: it reads files.
7828fn durable_terminal_history_of(
7829    terminal_ring: &Mutex<TerminalRing>,
7830    module_id: &str,
7831) -> subc_control::TerminalHistory {
7832    let read = terminal_ring
7833        .lock()
7834        .unwrap_or_else(|p| p.into_inner())
7835        .capture_durable_history();
7836    read.read(module_id)
7837}
7838
7839fn record_terminal_with_detail(
7840    module_id: &str,
7841    terminal_ring: &Arc<Mutex<TerminalRing>>,
7842    spawn_events: &SpawnEventFeed,
7843    exit_report: &ExitReport,
7844    disposition: TerminalDisposition,
7845    disposition_detail: Option<String>,
7846) {
7847    #[cfg(test)]
7848    terminal_history_tests::before_terminal_record(spawn_events);
7849    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
7850    let record = TerminalRecord {
7851        exit_code: exit_report.code,
7852        exit_signal: exit_report.signal,
7853        at_ms: exit_report.at_ms,
7854        disposition,
7855        exit_kind: exit_report.kind.into(),
7856        disposition_detail,
7857    };
7858    terminal_ring
7859        .lock()
7860        .unwrap_or_else(|poisoned| poisoned.into_inner())
7861        .record_exit(module_id, record);
7862}
7863
7864fn untrack_if_registration_released(
7865    process_liveness: &SupervisorProcessLiveness,
7866    registry: &Registry,
7867    module_id: &str,
7868    snapshot: &SharedSnapshot,
7869) {
7870    match registry.get_module(module_id) {
7871        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
7872        Ok(Some(_)) => {}
7873        Err(err) => {
7874            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
7875        }
7876    }
7877}
7878
7879/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
7880/// then apply the module's configured entries minus daemon-private capture keys.
7881///
7882/// Separated from `spawn_child` only so it can be asserted without spawning a
7883/// process — a duplicate of this logic in a test would pass while the real one
7884/// drifted, which is the defect class this function exists to avoid.
7885/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
7886/// nonce. A `protocol: "none"` module gets neither, because it cannot use
7887/// either and the argument would stop a stock binary from starting at all.
7888/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
7889///
7890/// The plain-spawn form, kept for the tests that assert its plan; spawns go
7891/// through [`apply_wire_spawn_args_for_role`].
7892#[cfg(test)]
7893fn apply_wire_spawn_args(
7894    command: &mut Command,
7895    spec: &ModuleSpec,
7896    connection_file_path: Option<&std::path::Path>,
7897    handle: Option<&SupervisorHandle>,
7898) -> Result<Option<NonceHandoff>, SuperviseError> {
7899    apply_wire_spawn_args_for_role(
7900        command,
7901        spec,
7902        connection_file_path,
7903        handle,
7904        SpawnRole::Plain,
7905    )
7906}
7907
7908/// The read end of a spawn's launch-nonce pipe, prepared by
7909/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
7910/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
7911/// server before spawn and authenticates its client after the child exists.
7912#[cfg(unix)]
7913type NonceHandoff = subc_os::LaunchNonceHandoff;
7914#[cfg(windows)]
7915type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
7916#[cfg(not(any(unix, windows)))]
7917type NonceHandoff = std::convert::Infallible;
7918
7919/// Create the launch nonce that a new module process must present when it
7920/// registers or opens a consumer connection, including a replacement process
7921/// started alongside a still-serving module.
7922///
7923/// A plain spawn replaces the module's recorded nonce. A swap candidate records
7924/// a separate candidate token so the still-serving incumbent and its consumers
7925/// keep their nonce. Both records are installed before the process exists, so
7926/// the child's initial HELLO registration cannot arrive ahead of its nonce.
7927///
7928/// On Unix the nonce is delivered only through a pipe. It is written into
7929/// a pipe whose read end the child gets as descriptor 3, named by
7930/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
7931/// process of the same user cannot read it with `ps eww`. That handoff is
7932/// returned rather than installed here, because installing it replaces
7933/// whatever the child has at descriptor 3 and so must be the last pre-exec
7934/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
7935/// the Linux callback that writes the process into its cgroup. Only after that
7936/// does `handoff.install_last` register the descriptor-3 replacement, so it
7937/// cannot close a descriptor the cgroup callback still needs.
7938/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
7939fn apply_wire_spawn_args_for_role(
7940    command: &mut Command,
7941    spec: &ModuleSpec,
7942    connection_file_path: Option<&std::path::Path>,
7943    handle: Option<&SupervisorHandle>,
7944    role: SpawnRole,
7945) -> Result<Option<NonceHandoff>, SuperviseError> {
7946    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
7947    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
7948    // included: a daemon started from a module's process tree inherits it,
7949    // and passing it on would point the child at a descriptor it does not
7950    // have.
7951    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
7952    #[cfg(windows)]
7953    {
7954        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
7955        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
7956    }
7957    // Remove inherited or configured copies too: withholding must mean absent.
7958    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
7959    if spec.protocol == ModuleProtocol::None {
7960        return Ok(None);
7961    }
7962    if let Some(connection_file_path) = connection_file_path {
7963        command.arg(SUBC_ARG).arg(connection_file_path);
7964    }
7965
7966    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
7967    // route.open attestation. Reserved modules additionally use the same nonce
7968    // for HELLO id-squatting protection. A respawn rotates both records.
7969    let nonce = generate_launch_nonce()?;
7970    if let Some(handle) = handle {
7971        match role {
7972            SpawnRole::Plain => {
7973                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
7974                if spec.reserved {
7975                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
7976                }
7977            }
7978            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
7979        }
7980    }
7981    #[cfg(unix)]
7982    let handoff = {
7983        let handoff =
7984            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
7985                program: spec.program.clone(),
7986                source,
7987                cgroup_path: None,
7988            })?;
7989        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
7990        Some(handoff)
7991    };
7992    #[cfg(windows)]
7993    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
7994    #[cfg(not(any(unix, windows)))]
7995    let handoff = None;
7996    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
7997    // until the daemon's per-spawn delivery records show that every running
7998    // Windows module has read its secret from the named pipe.
7999    #[cfg(not(unix))]
8000    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8001    Ok(handoff)
8002}
8003
8004fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8005    command.env_remove(CK_LOG_ENV);
8006    // The spawn role is the supervisor's to set, and only on a swap candidate
8007    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8008    // it, is what makes it absent on a plain spawn: the daemon's own
8009    // environment could carry it, and so could a spec built outside daemon
8010    // config (config refuses it as an `env` key). A module reading it on a
8011    // plain restart would pick the long swap budget and leave callers waiting.
8012    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8013    for (key, value) in &spec.env {
8014        // cortexkit-log currently exposes retention only as a Rust struct, not
8015        // environment names. These values are daemon-private sink metadata and
8016        // must never become a public child-process contract by being inherited.
8017        if matches!(
8018            key.as_str(),
8019            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8020        ) || key == SUBC_SPAWN_ROLE_ENV
8021        {
8022            continue;
8023        }
8024        command.env(key, value);
8025    }
8026}
8027
8028#[cfg(windows)]
8029fn prepare_windows_nonce_pipe(
8030    command: &mut Command,
8031    spec: &ModuleSpec,
8032    nonce: &str,
8033) -> Option<NonceHandoff> {
8034    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8035        Ok(handoff) => {
8036            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8037            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8038            // still register until every module reads the pipe. A reader without
8039            // this variable set to env returns the pipe error instead of using
8040            // an inherited or configured environment copy.
8041            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8042            Some(handoff)
8043        }
8044        Err(error) => {
8045            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8046            None
8047        }
8048    }
8049}
8050
8051#[cfg(windows)]
8052fn start_windows_nonce_pipe(
8053    handoff: Option<NonceHandoff>,
8054    pid: u32,
8055    spec: &ModuleSpec,
8056) -> Option<subc_os::LaunchNoncePipeDelivery> {
8057    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8058    // same maximum wait used for registration during a reload. Older modules
8059    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8060    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8061        Ok(delivery) => Some(delivery),
8062        Err(error) => {
8063            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8064            None
8065        }
8066    })
8067}
8068
8069fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8070    #[cfg(windows)]
8071    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8072        return Some(
8073            if snapshot
8074                .pipe_consumed
8075                .as_ref()
8076                .is_some_and(|consumed| consumed.get().is_some())
8077            {
8078                "pipe"
8079            } else {
8080                "env"
8081            }
8082            .to_owned(),
8083        );
8084    }
8085    let _ = snapshot;
8086    None
8087}
8088
8089/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8090/// of a blue/green swap.
8091#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8092enum SpawnRole {
8093    Plain,
8094    SwapCandidate,
8095}
8096
8097/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8098/// `apply_child_env` has already removed the variable for every spawn.
8099fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8100    if role == SpawnRole::SwapCandidate {
8101        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8102    }
8103}
8104
8105fn previous_instance_error(
8106    spec: &ModuleSpec,
8107    pid: u32,
8108    reason: crate::live_children::RetainedReason,
8109) -> SuperviseError {
8110    SuperviseError::Spawn {
8111        program: spec.program.clone(),
8112        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8113        cgroup_path: None,
8114    }
8115}
8116
8117fn spawn_child(
8118    spec: &ModuleSpec,
8119    connection_file_path: Option<&std::path::Path>,
8120    handle: Option<&SupervisorHandle>,
8121    ring: &Arc<Mutex<StderrRing>>,
8122    capture_logs_dir: Option<&std::path::Path>,
8123    roster: &ChildRoster,
8124    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8125) -> Result<SupervisedChild, SuperviseError> {
8126    spawn_child_in_slot(
8127        spec,
8128        connection_file_path,
8129        handle,
8130        ring,
8131        capture_logs_dir,
8132        roster,
8133        #[cfg(target_os = "linux")]
8134        cgroup_placement,
8135        SpawnRole::Plain,
8136        false,
8137    )
8138}
8139
8140/// Spawn one process of `spec` into a slot.
8141///
8142/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
8143/// A swap candidate needs a different cgroup from the process it is replacing,
8144/// which is still alive: in the same cgroup the two would be one kill domain,
8145/// and killing a failed candidate could take the incumbent with it.
8146///
8147/// The stderr capture file is `<module_id>.stderr.log` for every process of
8148/// the module, whichever slot it is in, because that is the one file
8149/// `ck module logs` reads. During a swap's overlap both processes append to it;
8150/// the daemon writes whole lines, so the two interleave by line, which is also
8151/// the merged view an operator wants while a swap runs.
8152#[allow(clippy::too_many_arguments)]
8153fn spawn_child_in_slot(
8154    spec: &ModuleSpec,
8155    connection_file_path: Option<&std::path::Path>,
8156    handle: Option<&SupervisorHandle>,
8157    ring: &Arc<Mutex<StderrRing>>,
8158    capture_logs_dir: Option<&std::path::Path>,
8159    roster: &ChildRoster,
8160    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8161    role: SpawnRole,
8162    alternate_slot: bool,
8163) -> Result<SupervisedChild, SuperviseError> {
8164    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8165        return Err(previous_instance_error(spec, pid, reason));
8166    }
8167    if roster.is_closed() {
8168        return Err(SuperviseError::Spawn {
8169            program: spec.program.clone(),
8170            source: io::Error::other("the daemon is shutting down; not starting a new process"),
8171            cgroup_path: None,
8172        });
8173    }
8174    #[cfg(target_os = "linux")]
8175    let cgroup_name = {
8176        // Slot names alone are not kill domains: a retired incumbent may still
8177        // be draining when a later enable/restart spawns into the same slot.
8178        // Decimal entropy keeps the suffix unambiguous; Placement performs
8179        // the module-id escaping and constructs the filesystem path.
8180        if cgroup_placement.is_none() {
8181            swap::cgroup_name(&spec.module_id, alternate_slot)
8182        } else {
8183            let nonce = generate_launch_nonce()?;
8184            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8185            // Leave room for byte escaping and the suffix under NAME_MAX. The
8186            // label is only for humans; the nonce identifies the kill domain.
8187            let mut end = spec.module_id.len().min(64);
8188            while !spec.module_id.is_char_boundary(end) {
8189                end -= 1;
8190            }
8191            format!(
8192                "{}_{suffix}",
8193                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8194            )
8195        }
8196    };
8197    #[cfg(not(target_os = "linux"))]
8198    let _ = alternate_slot;
8199    #[cfg(target_os = "macos")]
8200    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8201    #[cfg(windows)]
8202    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8203        program: spec.program.clone(),
8204        source,
8205        cgroup_path: None,
8206    })?;
8207    #[cfg(windows)]
8208    let mut command = Command::new(image_capture.path());
8209    #[cfg(not(any(target_os = "macos", windows)))]
8210    let mut command = Command::new(&spec.program);
8211    command.args(&spec.args);
8212    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
8213    // that is the whole of the intent, so remove that one key rather than the
8214    // environment.
8215    //
8216    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
8217    // and took the POSIX environment with it. Modules spawned that way had no
8218    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
8219    // logging:
8220    //
8221    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
8222    //     both unset it fell back to the temp dir alone and `ck` could not find
8223    //     a daemon running on the same machine from inside any module's process
8224    //     tree — reporting a path the file has never lived at, which reads as
8225    //     "the daemon did not write its file".
8226    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
8227    //     the RELATIVE `.local/share`, so a module deriving its own store path
8228    //     resolved it against its own CWD. That is the store-fragmentation
8229    //     defect the daemon already refuses in config (`parse_doc` rejects a
8230    //     relative `storage.data_home`) arriving by derivation instead.
8231    //   * anything a module spawns inherited it: git without ~/.gitconfig,
8232    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
8233    //     quietly rather than erroring.
8234    //
8235    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
8236    // offered one candidate under /tmp while the file sat in /run/user/1000.
8237    //
8238    // A configured module is unaffected either way: `module_spec()` puts the
8239    // resolved CK_LOG into `spec.env`, which is applied below and therefore
8240    // wins over anything ambient.
8241    apply_child_env(&mut command, spec);
8242    apply_spawn_role(&mut command, role);
8243    let nonce_handoff =
8244        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8245
8246    #[cfg(target_os = "linux")]
8247    let cgroup_path = cgroup_placement
8248        .map(|placement| placement.module_path(&cgroup_name))
8249        .transpose()
8250        .map_err(|source| SuperviseError::Cgroup {
8251            module_id: spec.module_id.clone(),
8252            source,
8253        })?;
8254    #[cfg(not(target_os = "linux"))]
8255    let cgroup_path: Option<PathBuf> = None;
8256    #[cfg(target_os = "linux")]
8257    if let Some(path) = &cgroup_path {
8258        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8259            if let Some(placement) = cgroup_placement {
8260                remove_module_cgroup(placement, &cgroup_name);
8261            }
8262            return Err(error);
8263        }
8264    }
8265
8266    let output_sink = if let Some(logs_dir) = capture_logs_dir {
8267        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8268        match ChildOutputSink::open(&path, capture_retention(spec)) {
8269            Ok(sink) => sink,
8270            Err(error) => {
8271                warn!(
8272                    module_id = %spec.module_id,
8273                    path = %path.display(),
8274                    error = %error,
8275                    "could not open child output capture file; forwarding to stderr"
8276                );
8277                ChildOutputSink::Stderr
8278            }
8279        }
8280    } else {
8281        ChildOutputSink::Stderr
8282    };
8283
8284    command.stdout(Stdio::piped());
8285    command.stderr(Stdio::piped());
8286    command.kill_on_drop(true);
8287    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
8288    // before exec). In the daemon's group, a service manager that kills the
8289    // job's process group when the daemon exits (launchd's default) killed
8290    // every module at the same moment its control connection closed, so no
8291    // module ever ran its EOF teardown on a daemon stop. Outside that group a
8292    // module is reached only by the daemon: the EOF it sees when its
8293    // connection closes, and the bounded stop in `child_roster` for anything
8294    // still running after that. On Linux this composes with the cgroup
8295    // placement above: that is a pre_exec write to cgroup.procs, std performs
8296    // setpgid in the child before running pre_exec callbacks, and the two
8297    // change independent process attributes.
8298    //
8299    // stdin is /dev/null because a process outside the terminal's foreground
8300    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
8301    // by hand would otherwise hand down. Under a service manager stdin is
8302    // already /dev/null.
8303    #[cfg(unix)]
8304    command.process_group(0);
8305    command.stdin(Stdio::null());
8306    // The LAST pre-exec step, after the cgroup placement above: installing the
8307    // nonce at descriptor 3 replaces whatever the child had there, which could
8308    // be the descriptor an earlier step writes through.
8309    #[cfg(unix)]
8310    if let Some(handoff) = nonce_handoff {
8311        handoff.install_last(command.as_std_mut());
8312    }
8313    #[cfg(not(any(unix, windows)))]
8314    let _ = nonce_handoff;
8315
8316    // Containment, step 1 of 3 (issue #109): create the child suspended so it
8317    // cannot run a single instruction -- and therefore cannot spawn a
8318    // grandchild -- before it is in the job. See `contain_spawned_child` for the
8319    // other two steps and why the window matters.
8320    #[cfg(windows)]
8321    {
8322        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8323            program: spec.program.clone(),
8324            source,
8325            cgroup_path: None,
8326        })?;
8327        subc_jobobject::suspend_on_create_async(&mut command);
8328    }
8329    let mut child = match command.spawn() {
8330        Ok(child) => child,
8331        Err(source) => {
8332            #[cfg(target_os = "linux")]
8333            if let Some(placement) = cgroup_placement {
8334                remove_module_cgroup(placement, &cgroup_name);
8335            }
8336            return Err(SuperviseError::Spawn {
8337                program: spec.program.clone(),
8338                source,
8339                cgroup_path,
8340            });
8341        }
8342    };
8343    #[cfg(test)]
8344    spawn_record_tests::after_create(&child, spec);
8345    // The parent must close its writer now: the acknowledgement pipe reports EOF
8346    // only when every writer is gone, and the child's copy closes when the
8347    // trampoline replaces itself with the module. Command holds only an integer
8348    // in its pre_exec callback, not another writer.
8349    #[cfg(target_os = "macos")]
8350    drop(exec_ack);
8351
8352    // Bind while the child is still suspended, before job assignment resumes it.
8353    // Keeping the executable handle open across CreateProcessW denies rename
8354    // and deletion, so the file cannot be exchanged during spawn. Bind that
8355    // file ID to the process handle returned by spawn, never a reopened PID.
8356    #[cfg(windows)]
8357    let windows_image = match image_capture.bind_tokio(&child) {
8358        Ok(image) => Arc::new(image),
8359        Err(source) => {
8360            kill_and_reap_rejected_child(&mut child);
8361            return Err(SuperviseError::Spawn {
8362                program: spec.program.clone(),
8363                source,
8364                cgroup_path: None,
8365            });
8366        }
8367    };
8368    // Assign the suspended child to its module job; resume only after its
8369    // process identity has been durably recorded for orphan recovery.
8370    #[cfg(windows)]
8371    let job = contain_spawned_child(&mut child, spec)?;
8372    let spawned_at_ms = unix_ms_now();
8373    #[cfg(not(windows))]
8374    let spawned_from = spec.program.clone();
8375    #[cfg(windows)]
8376    let spawned_from = windows_image.path().to_path_buf();
8377    #[cfg(not(windows))]
8378    let spawned_file_identity = spawned_file_identity(&spawned_from);
8379    #[cfg(windows)]
8380    let spawned_file_identity = Some(SpawnedFileIdentity {
8381        device: 0,
8382        inode: 0,
8383        windows_image: Some(Arc::clone(&windows_image)),
8384    });
8385    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8386        program: spec.program.clone(),
8387        source: io::Error::other("spawned child exposed no live pid"),
8388        cgroup_path: cgroup_path.clone(),
8389    })?;
8390    #[cfg(windows)]
8391    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8392    #[cfg(not(windows))]
8393    let process_start_time = crate::provenance::process_start_time(pid);
8394    #[cfg(windows)]
8395    let process_start_time = Some(windows_image.start_time());
8396    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8397    #[cfg(all(test, target_os = "macos"))]
8398    privacy_exec_boundary_tests::before_image_sample(spec, pid);
8399    // Record the committed kernel image, not the configured pathname: PATH
8400    // lookup and shebang interpretation can select a different executable.
8401    // Linux's posix_spawn can return before that image is installed, so the
8402    // observer also waits for the kernel's first-exec flag to clear.
8403    #[cfg(not(windows))]
8404    let recorded_image = observe_spawned_image(pid);
8405    // spawn() confirms only the first exec, into the trampoline. Never persist
8406    // the trampoline image; the asynchronous acknowledgement publishes the
8407    // module image once the trampoline has replaced itself with the module.
8408    #[cfg(target_os = "macos")]
8409    let recorded_image = if privacy_exec.is_some() {
8410        None
8411    } else {
8412        recorded_image
8413    };
8414    #[cfg(target_os = "linux")]
8415    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8416    #[cfg(not(target_os = "linux"))]
8417    let recorded_cgroup_name = None;
8418    let roster_guard = roster.admit(
8419        spec.module_id.clone(),
8420        pid,
8421        spec.protocol,
8422        process_start_time,
8423        crate::child_roster::RecordedIdentity {
8424            #[cfg(not(windows))]
8425            start_time: recorded_image.map(|image| image.start_time),
8426            #[cfg(windows)]
8427            start_time: process_start_time,
8428            #[cfg(not(windows))]
8429            executable: recorded_image
8430                .and_then(|image| image.executable)
8431                .map(crate::live_children::ExecutableIdentity::from),
8432            #[cfg(windows)]
8433            executable: Some(crate::live_children::ExecutableIdentity::from(
8434                windows_image.identity(),
8435            )),
8436            cgroup_name: recorded_cgroup_name,
8437            #[cfg(target_os = "linux")]
8438            cgroup_placement: cgroup_placement.cloned(),
8439        },
8440    );
8441    let roster_guard = match roster_guard {
8442        Ok(guard) => guard,
8443        Err(source) => {
8444            #[cfg(windows)]
8445            if let Some(job) = &job {
8446                let _ = job.terminate();
8447            }
8448            #[cfg(target_os = "linux")]
8449            kill_module_cgroup(cgroup_placement, &cgroup_name);
8450            kill_and_reap_rejected_child(&mut child);
8451            #[cfg(target_os = "linux")]
8452            if let Some(placement) = cgroup_placement {
8453                remove_module_cgroup(placement, &cgroup_name);
8454            }
8455            return Err(SuperviseError::Spawn {
8456                program: spec.program.clone(),
8457                source: io::Error::other(format!(
8458                    "could not durably record module child: {source}"
8459                )),
8460                cgroup_path,
8461            });
8462        }
8463    };
8464    // The check at the top of this function can pass just before daemon
8465    // shutdown begins, and the process is only in the roster from here on.
8466    // The shutdown stop returns as soon as it finds the roster empty, so a
8467    // process admitted after that look would outlive the daemon. The roster
8468    // is closed before the stop first reads it and admission happens under
8469    // the roster's lock, so either the stop sees this process or this check
8470    // sees the roster closed: end the process now rather than start a module
8471    // the daemon is about to stop.
8472    if roster.is_closed() {
8473        // This child was never admitted, so there is no module protocol shutdown to wait for.
8474        #[cfg(target_os = "linux")]
8475        kill_module_cgroup(cgroup_placement, &cgroup_name);
8476        if let Err(error) = child.start_kill() {
8477            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8478        }
8479        #[cfg(target_os = "linux")]
8480        if let Some(placement) = cgroup_placement {
8481            // This spawn was never admitted, so shutdown has no roster entry
8482            // to await. Do not detach its cleanup: the runtime could exit
8483            // before that task reaps the rejected child and removes its group.
8484            while matches!(child.try_wait(), Ok(None)) {
8485                std::thread::yield_now();
8486            }
8487            if matches!(
8488                subc_cgroup::kill_module(Some(placement), &cgroup_name),
8489                subc_cgroup::KillOutcome::Killed
8490            ) {
8491                if let Ok(path) = placement.module_path(&cgroup_name) {
8492                    while std::fs::read_to_string(path.join("cgroup.events"))
8493                        .ok()
8494                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8495                    {
8496                        std::thread::yield_now();
8497                    }
8498                }
8499            }
8500            remove_module_cgroup(placement, &cgroup_name);
8501        }
8502        drop(roster_guard);
8503        return Err(SuperviseError::Spawn {
8504            program: spec.program.clone(),
8505            source: io::Error::other(
8506                "the daemon began shutting down while this process was starting; ended it",
8507            ),
8508            cgroup_path,
8509        });
8510    }
8511
8512    #[cfg(windows)]
8513    if let Err(error) = resume_suspended_child(pid, spec) {
8514        kill_and_reap_rejected_child(&mut child);
8515        return Err(error);
8516    }
8517
8518    let stdout_pump = match child.stdout.take() {
8519        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8520        None => {
8521            warn!(
8522                module_id = %spec.module_id,
8523                "spawned child exposed no stdout pipe; file capture will be incomplete"
8524            );
8525            None
8526        }
8527    };
8528    let stderr_pump = match child.stderr.take() {
8529        Some(stderr) => {
8530            let generation = ring
8531                .lock()
8532                .unwrap_or_else(|poisoned| poisoned.into_inner())
8533                .begin_process();
8534            Some(StderrPump {
8535                task: tokio::spawn(pump_stderr_to(
8536                    stderr,
8537                    Arc::clone(ring),
8538                    generation,
8539                    output_sink,
8540                )),
8541                generation,
8542            })
8543        }
8544        None => {
8545            // Spawning succeeded but the pipe did not materialise. Recording it as
8546            // uncaptured keeps the tail honest: the alternative is an empty tail
8547            // that reads as a module which printed nothing.
8548            ring.lock()
8549                .unwrap_or_else(|poisoned| poisoned.into_inner())
8550                .mark_not_captured("stderr pipe was not available on spawn");
8551            warn!(
8552                module_id = %spec.module_id,
8553                "spawned child exposed no stderr pipe; tail will be unavailable"
8554            );
8555            None
8556        }
8557    };
8558
8559    let exec = Arc::new(crate::activity::ExecOutcome::new());
8560    #[cfg(target_os = "macos")]
8561    let pending_exec = privacy_exec.is_some();
8562    #[cfg(not(target_os = "macos"))]
8563    let pending_exec = false;
8564    if !pending_exec {
8565        exec.finish(crate::activity::ExecState::CommittedExec);
8566    }
8567    Ok(SupervisedChild {
8568        child,
8569        exec: crate::activity::ExecOwner(exec),
8570        #[cfg(windows)]
8571        nonce_delivery,
8572        protocol: spec.protocol,
8573        #[cfg(target_os = "linux")]
8574        module_id: cgroup_name,
8575        #[cfg(target_os = "linux")]
8576        cgroup_placement: cgroup_placement.cloned(),
8577        #[cfg(windows)]
8578        job,
8579        stdout_pump,
8580        stderr_pump,
8581        stderr_ring: Arc::clone(ring),
8582        spawned_at_ms,
8583        spawned_from,
8584        spawned_file_identity,
8585        process_start_time,
8586        process_identity,
8587        pid,
8588        roster_guard: Some(roster_guard),
8589        #[cfg(target_os = "macos")]
8590        privacy_exec,
8591        #[cfg(target_os = "macos")]
8592        report_ready: Arc::new(OnceLock::new()),
8593        spawn_failure: None,
8594    })
8595}
8596
8597#[cfg(test)]
8598mod spawn_record_tests {
8599    use super::*;
8600
8601    thread_local! {
8602        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8603    }
8604
8605    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8606        LAST_PID.set(child.id());
8607        #[cfg(windows)]
8608        if let Some((_, address)) = _spec
8609            .env
8610            .iter()
8611            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8612        {
8613            use std::io::{Read, Write};
8614            let mut stream = std::net::TcpStream::connect(address).unwrap();
8615            stream
8616                .write_all(&child.id().unwrap().to_ne_bytes())
8617                .unwrap();
8618            let mut release = [0];
8619            stream.read_exact(&mut release).unwrap();
8620        }
8621    }
8622
8623    #[cfg(windows)]
8624    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8625        if spec
8626            .env
8627            .iter()
8628            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8629        {
8630            Err(io::Error::other("injected module job assignment failure"))
8631        } else {
8632            Ok(())
8633        }
8634    }
8635
8636    fn spec() -> ModuleSpec {
8637        ModuleSpec {
8638            module_id: "spawn-record-worker".into(),
8639            #[cfg(unix)]
8640            program: "/bin/sleep".into(),
8641            #[cfg(windows)]
8642            program: std::env::current_exe().unwrap(),
8643            #[cfg(unix)]
8644            args: vec!["60".into()],
8645            #[cfg(windows)]
8646            args: vec![
8647                "--exact".into(),
8648                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
8649                "--nocapture".into(),
8650            ],
8651            env: vec![],
8652            reserved: false,
8653            reserved_prefixes: vec![],
8654            protocol: ModuleProtocol::None,
8655            overlap: Default::default(),
8656        }
8657    }
8658
8659    #[tokio::test]
8660    async fn record_sync_failure_fails_spawn_and_reaps_child() {
8661        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
8662        let supervisor =
8663            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8664                .with_live_children_record(home.join("live-children.json"));
8665        crate::live_children::fail_next_file_sync();
8666        let result = supervisor.spawn(spec());
8667        let failed = result.is_err();
8668        if let Ok(module) = result {
8669            module.stop().await.unwrap();
8670        }
8671        assert!(failed, "record failure must fail spawn");
8672        let pid = LAST_PID.get().expect("the real OS child was created");
8673        assert!(
8674            subc_os::Process::open(pid).unwrap().is_none(),
8675            "child {pid} survived rejected admission or was not reaped"
8676        );
8677    }
8678
8679    #[cfg(target_os = "macos")]
8680    #[test]
8681    #[ignore = "manual on-device latency measurement"]
8682    fn durable_record_write_latency_50() {
8683        let home = cortexkit_test_support::ScratchDir::new("record-latency");
8684        let path = home.join("live-children.json");
8685        let children = vec![crate::live_children::LiveChild {
8686            module_id: "latency-worker".into(),
8687            pid: std::process::id(),
8688            protocol: ModuleProtocol::None,
8689            start_time: Some(1),
8690            executable: None,
8691            cgroup_name: None,
8692        }];
8693        let mut samples = Vec::new();
8694        for _ in 0..50 {
8695            let start = std::time::Instant::now();
8696            crate::live_children::write_record(&path, &children).unwrap();
8697            samples.push(start.elapsed());
8698        }
8699        samples.sort();
8700        eprintln!(
8701            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
8702            (samples[24] + samples[25]) / 2,
8703            samples[47]
8704        );
8705    }
8706
8707    #[cfg(windows)]
8708    #[test]
8709    fn windows_store_touch_fixture() {
8710        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
8711            std::fs::write(path, b"module ran").unwrap();
8712        }
8713    }
8714
8715    #[cfg(windows)]
8716    #[tokio::test]
8717    async fn windows_daemon_fixture() {
8718        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
8719            return;
8720        };
8721        let mut spec = spec();
8722        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
8723        let supervisor =
8724            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8725        let _module = supervisor.spawn(spec).unwrap();
8726    }
8727
8728    #[cfg(windows)]
8729    #[test]
8730    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
8731        use std::io::Read;
8732        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
8733        // An outer job models a scheduler/service host. The daemon's root and
8734        // root job must nest without requiring breakaway from this job.
8735        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8736        command
8737            .args([
8738                "--exact",
8739                "supervise::spawn_record_tests::windows_daemon_fixture",
8740                "--nocapture",
8741            ])
8742            .env(
8743                "SUBC_TEST_DAEMON_BARRIER",
8744                listener.local_addr().unwrap().to_string(),
8745            );
8746        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8747        listener.set_nonblocking(true).unwrap();
8748        let deadline = std::time::Instant::now() + Duration::from_secs(30);
8749        let mut stream = loop {
8750            if let Ok((stream, _)) = listener.accept() {
8751                break stream;
8752            }
8753            assert!(
8754                daemon.child.try_wait().unwrap().is_none(),
8755                "daemon failed before rendezvous"
8756            );
8757            assert!(
8758                std::time::Instant::now() < deadline,
8759                "daemon did not reach CreateProcess failpoint"
8760            );
8761            std::thread::yield_now();
8762        };
8763        stream
8764            .set_read_timeout(Some(Duration::from_secs(30)))
8765            .unwrap();
8766        let mut pid = [0; 4];
8767        stream.read_exact(&mut pid).unwrap();
8768        let pid = u32::from_ne_bytes(pid);
8769        assert!(subc_jobobject::process_exists(pid));
8770        // Terminate only the daemon, not its outer job: otherwise the host job
8771        // would also kill a child whose daemon never joined its root job.
8772        daemon.child.kill().unwrap();
8773        daemon.child.wait().unwrap();
8774        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
8775        if !exited {
8776            daemon.job.terminate().unwrap();
8777        }
8778        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
8779    }
8780
8781    #[cfg(windows)]
8782    #[tokio::test]
8783    async fn windows_containment_failure_never_resumes() {
8784        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
8785        let sentinel = home.join("store-touched");
8786        let mut spec = spec();
8787        spec.env
8788            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
8789        spec.env.push((
8790            "SUBC_TEST_STORE_TOUCH".into(),
8791            sentinel.display().to_string(),
8792        ));
8793        let supervisor =
8794            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8795        let result = supervisor.spawn(spec);
8796        // A version that resumes despite assignment failure must have time to
8797        // execute its sentinel before dropping the returned module kills it.
8798        if let Ok(module) = result {
8799            let _ = tokio::time::timeout(Duration::from_secs(10), async {
8800                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
8801                    tokio::task::yield_now().await;
8802                }
8803            })
8804            .await;
8805            let _ = module.stop().await;
8806            panic!(
8807                "assignment failure resumed a module; store touched={}",
8808                sentinel.exists()
8809            );
8810        }
8811        assert!(
8812            !sentinel.exists(),
8813            "refused module executed store-touch sentinel"
8814        );
8815        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
8816    }
8817
8818    #[cfg(windows)]
8819    #[tokio::test]
8820    async fn windows_nested_daemon_fixture() {
8821        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
8822            return;
8823        };
8824        let mut spec = spec();
8825        spec.env.push((
8826            "SUBC_TEST_STORE_TOUCH".into(),
8827            sentinel.to_str().unwrap().into(),
8828        ));
8829        let supervisor =
8830            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8831        let module = supervisor
8832            .spawn(spec)
8833            .expect("per-module job must nest below daemon and host jobs");
8834        tokio::time::timeout(Duration::from_secs(10), async {
8835            while !PathBuf::from(&sentinel).exists() {
8836                tokio::task::yield_now().await;
8837            }
8838        })
8839        .await
8840        .unwrap();
8841        module.stop().await.unwrap();
8842    }
8843
8844    #[cfg(windows)]
8845    #[test]
8846    fn windows_module_job_nests_under_host_job() {
8847        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
8848        let sentinel = home.join("nested-store-touched");
8849        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8850        command
8851            .args([
8852                "--exact",
8853                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
8854                "--nocapture",
8855            ])
8856            .env("SUBC_TEST_NESTED_STORE", &sentinel);
8857        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8858        assert!(daemon.child.wait().unwrap().success());
8859        assert!(sentinel.exists(), "nested protocol:none module never ran");
8860    }
8861}
8862
8863#[cfg(target_os = "linux")]
8864pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
8865    use subc_cgroup::KillOutcome;
8866    match subc_cgroup::kill_module(placement, module_id) {
8867        KillOutcome::Killed => {}
8868        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
8869            debug!(
8870                module_id,
8871                "cgroup tree kill unavailable; using direct-child kill"
8872            );
8873        }
8874        KillOutcome::IoError { path, error } => {
8875            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
8876        }
8877    }
8878}
8879
8880/// Assign a freshly spawned Windows child without ever resuming on failure.
8881///
8882/// Steps 2 and 3 of the suspended-create contract: the job is created and the
8883/// child assigned **while it is still suspended** (step 1 is
8884/// `suspend_on_create_async` at the spawn site). The caller resumes only after
8885/// the child's roster admission is durable. Failed containment kills and reaps
8886/// the suspended child instead of running an unprotected module.
8887#[cfg(windows)]
8888fn contain_spawned_child(
8889    child: &mut Child,
8890    spec: &ModuleSpec,
8891) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
8892    let assignment = || {
8893        let job = subc_jobobject::JobObject::new()?;
8894        #[cfg(test)]
8895        spawn_record_tests::assignment_failpoint(spec)?;
8896        job.assign(child)?;
8897        Ok::<_, io::Error>(job)
8898    };
8899    match assignment() {
8900        Ok(job) => Ok(Some(job)),
8901        Err(source) => {
8902            kill_and_reap_rejected_child(child);
8903            Err(SuperviseError::Spawn {
8904                program: spec.program.clone(),
8905                source: io::Error::other(format!(
8906                    "Windows module job containment failed: {source}"
8907                )),
8908                cgroup_path: None,
8909            })
8910        }
8911    }
8912}
8913
8914/// Rejection is synchronous: do not return a spawn error while the child can
8915/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
8916fn kill_and_reap_rejected_child(child: &mut Child) {
8917    #[cfg(unix)]
8918    if let Some(pid) = child
8919        .id()
8920        .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
8921    {
8922        let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
8923    }
8924    let _ = child.start_kill();
8925    while matches!(child.try_wait(), Ok(None)) {
8926        std::thread::yield_now();
8927    }
8928}
8929
8930/// Resume a suspended child, killing it if it cannot be started.
8931///
8932/// A suspended process holds a pid and does nothing, so there is no useful
8933/// state to return: the caller gets an error and the spawn fails.
8934#[cfg(windows)]
8935fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
8936    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
8937        // Kill it here rather than leaving a suspended process for the caller
8938        // to notice; `kill_on_drop` would eventually do this, but the module
8939        // would have been reported as running in between.
8940        let _ = std::process::Command::new("taskkill.exe")
8941            .args(["/PID", &pid.to_string(), "/T", "/F"])
8942            .stdin(Stdio::null())
8943            .stdout(Stdio::null())
8944            .stderr(Stdio::null())
8945            .status();
8946        return Err(SuperviseError::Spawn {
8947            program: spec.program.clone(),
8948            source,
8949            cgroup_path: None,
8950        });
8951    }
8952    Ok(())
8953}
8954
8955#[cfg(target_os = "linux")]
8956fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
8957    match placement.remove_module(module_id) {
8958        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
8959        Err(error) => warn!(
8960            module_id,
8961            error = %error,
8962            "could not remove module cgroup after process exit; continuing teardown"
8963        ),
8964    }
8965}
8966
8967#[cfg(target_os = "linux")]
8968async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
8969    // Reaping the direct child is not proof its descendants exited. End the
8970    // residual tree and wait for the kernel's population fact before rmdir;
8971    // otherwise a successful parent wait leaks a directory on each restart.
8972    if matches!(
8973        subc_cgroup::kill_module(Some(placement), module_id),
8974        subc_cgroup::KillOutcome::Killed
8975    ) {
8976        if let Ok(path) = placement.module_path(module_id) {
8977            while std::fs::read_to_string(path.join("cgroup.events"))
8978                .ok()
8979                .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8980            {
8981                sleep(Duration::from_millis(1)).await;
8982            }
8983        }
8984    }
8985    remove_module_cgroup(placement, module_id);
8986}
8987
8988#[cfg(target_os = "linux")]
8989fn apply_cgroup_placement(
8990    command: &mut Command,
8991    spec: &ModuleSpec,
8992    path: &std::path::Path,
8993) -> Result<(), SuperviseError> {
8994    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
8995        module_id: spec.module_id.clone(),
8996        source,
8997    })
8998}
8999
9000fn capture_retention(spec: &ModuleSpec) -> Retention {
9001    let defaults = Retention::default();
9002    let value = |name: &str| {
9003        spec.env
9004            .iter()
9005            .rev()
9006            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9007    };
9008    Retention {
9009        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9010            .and_then(|value| value.parse().ok())
9011            .unwrap_or(defaults.max_file_mb),
9012        keep: value(CAPTURE_KEEP_ENV)
9013            .and_then(|value| value.parse().ok())
9014            .unwrap_or(defaults.keep),
9015        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9016            .and_then(|value| value.parse().ok())
9017            .unwrap_or(defaults.max_age_days),
9018    }
9019}
9020
9021/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
9022/// module's registration to the exact process the supervisor spawned.
9023fn generate_launch_nonce() -> Result<String, SuperviseError> {
9024    let mut bytes = [0u8; 32];
9025    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9026        reason: source.to_string(),
9027    })?;
9028    let mut hex = String::with_capacity(64);
9029    for b in bytes {
9030        use std::fmt::Write;
9031        let _ = write!(hex, "{b:02x}");
9032    }
9033    Ok(hex)
9034}
9035
9036/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
9037/// signal about how many leading bytes matched.
9038fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9039    if a.len() != b.len() {
9040        return false;
9041    }
9042    let mut diff = 0u8;
9043    for (x, y) in a.iter().zip(b.iter()) {
9044        diff |= x ^ y;
9045    }
9046    diff == 0
9047}
9048
9049/// The kernel's image after an acknowledged exec, shared by ordinary launches
9050/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
9051/// not identities inferred from a configured pathname.
9052#[cfg(not(windows))]
9053fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9054    let process = subc_os::Process::open(pid).ok().flatten()?;
9055    #[cfg(target_os = "linux")]
9056    return sample_after_linux_exec(
9057        || {
9058            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9059            let committed = linux_exec_committed(&stat);
9060            #[cfg(test)]
9061            spawn_image_tests::report_exec_state(committed);
9062            committed
9063        },
9064        || process.observe(),
9065    );
9066    #[cfg(not(target_os = "linux"))]
9067    process.observe()
9068}
9069
9070/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
9071/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
9072/// installation. Until it clears, /proc/pid/exe can still name the daemon.
9073#[cfg(target_os = "linux")]
9074fn linux_exec_committed(stat: &str) -> Option<bool> {
9075    const PF_FORKNOEXEC: u64 = 0x40;
9076    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9077    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9078}
9079
9080#[cfg(target_os = "linux")]
9081fn sample_after_linux_exec(
9082    mut exec_committed: impl FnMut() -> Option<bool>,
9083    observe: impl FnOnce() -> Option<subc_os::Observation>,
9084) -> Option<subc_os::Observation> {
9085    let deadline = std::time::Instant::now() + Duration::from_secs(1);
9086    while !exec_committed()? {
9087        // A failed or indefinitely stalled exec must not publish the inherited
9088        // image as authority to signal a future orphan. Missing identity keeps
9089        // the existing conservative orphan-sweep behavior.
9090        if std::time::Instant::now() >= deadline {
9091            return None;
9092        }
9093        std::thread::sleep(Duration::from_micros(50));
9094    }
9095    observe()
9096}
9097
9098#[cfg(all(test, target_os = "linux"))]
9099mod spawn_image_tests {
9100    use super::*;
9101    use std::cell::{Cell, RefCell};
9102
9103    enum ExecEvent {
9104        Uncommitted,
9105        Image(Option<subc_os::Observation>),
9106    }
9107
9108    thread_local! {
9109        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9110    }
9111
9112    pub(super) fn report_exec_state(committed: Option<bool>) {
9113        if committed == Some(false) {
9114            EXEC_EVENTS.with(|events| {
9115                if let Some(events) = events.borrow_mut().take() {
9116                    let _ = events.send(ExecEvent::Uncommitted);
9117                }
9118            });
9119        }
9120    }
9121
9122    #[test]
9123    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9124        use std::io::{Read, Write};
9125        use std::os::fd::AsRawFd;
9126
9127        let descriptor = std::fs::File::open("/dev/null").unwrap();
9128        let mut command = std::process::Command::new("/bin/sleep");
9129        command.arg("30");
9130        let (mut ready, mut release) =
9131            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9132                .unwrap();
9133        let spawn = std::thread::spawn(move || command.spawn());
9134        let mut message = [0u8; 8];
9135        ready.read_exact(&mut message).unwrap();
9136        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9137        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9138        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9139        let (events, received) = std::sync::mpsc::channel();
9140        let observer = std::thread::spawn(move || {
9141            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9142            let image = observe_spawned_image(pid);
9143            let _ = events.send(ExecEvent::Image(image));
9144        });
9145        // Release only once the production observer has seen the uncommitted
9146        // exec, or has incorrectly returned the inherited image. Either way,
9147        // release and reap before asserting so a regression cannot leak a child.
9148        let first = received.recv_timeout(Duration::from_secs(5));
9149        release.write_all(&[1]).unwrap();
9150        let mut child = spawn.join().unwrap().unwrap();
9151        let after = match first {
9152            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9153                Ok(ExecEvent::Image(image)) => image,
9154                _ => None,
9155            },
9156            Ok(ExecEvent::Image(image)) => image,
9157            Err(_) => None,
9158        };
9159        observer.join().unwrap();
9160        child.kill().unwrap();
9161        child.wait().unwrap();
9162
9163        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9164        assert_eq!(
9165            inherited.unwrap().executable,
9166            subc_os::file_identity(&std::env::current_exe().unwrap())
9167        );
9168        assert_eq!(
9169            after.unwrap().executable,
9170            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9171        );
9172    }
9173
9174    #[test]
9175    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9176        let committed = Cell::new(false);
9177        let inherited = subc_os::Observation {
9178            start_time: 23,
9179            executable: Some(subc_os::FileIdentity {
9180                device: 29,
9181                inode: 31,
9182            }),
9183        };
9184        let module = subc_os::Observation {
9185            start_time: 23,
9186            executable: Some(subc_os::FileIdentity {
9187                device: 29,
9188                inode: 37,
9189            }),
9190        };
9191        let polls = Cell::new(0);
9192        let image = sample_after_linux_exec(
9193            || {
9194                polls.set(polls.get() + 1);
9195                // Hold the inherited image through the first poll, as in the
9196                // posix_spawn parent-wake window, then commit the module image.
9197                committed.set(polls.get() == 2);
9198                Some(committed.get())
9199            },
9200            || Some(if committed.get() { module } else { inherited }),
9201        );
9202        assert_eq!(image, Some(module));
9203        assert_eq!(polls.get(), 2);
9204    }
9205
9206    #[test]
9207    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9208        assert_eq!(
9209            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9210            Some(false)
9211        );
9212        assert_eq!(
9213            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9214            Some(true)
9215        );
9216        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9217        assert_eq!(linux_exec_committed("17 (a) R"), None);
9218    }
9219
9220    #[test]
9221    fn unreadable_exec_state_never_samples_an_inherited_image() {
9222        assert_eq!(
9223            sample_after_linux_exec(
9224                || None,
9225                || panic!("must not sample before exec confirmation")
9226            ),
9227            None
9228        );
9229    }
9230
9231    #[test]
9232    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9233        assert_eq!(
9234            sample_after_linux_exec(
9235                || Some(false),
9236                || panic!("must not sample after an exec timeout")
9237            ),
9238            None
9239        );
9240    }
9241}
9242
9243/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
9244/// different executable after the no-delete capture was taken.
9245#[cfg(windows)]
9246fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9247    let mut program = spec.program.clone();
9248    if program.is_absolute() || program.components().count() > 1 {
9249        // An explicit path can name a PE image with no extension. Try that
9250        // exact file first, as Command does, before falling back to .exe.
9251        match subc_os::ExecutableCapture::open(&program) {
9252            Ok(capture) => return Ok(capture),
9253            Err(error)
9254                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9255            Err(error) => return Err(error),
9256        }
9257        program.set_extension("exe");
9258        return subc_os::ExecutableCapture::open(&program);
9259    }
9260    if program.extension().is_none() {
9261        program.set_extension("exe");
9262    }
9263    let custom_path = spec
9264        .env
9265        .iter()
9266        .rev()
9267        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9268    let mut directories = Vec::new();
9269    // These are Windows' default executable search locations. A configured
9270    // PATH still overrides the inherited PATH, not the system directories.
9271    if let Some(parent) = std::env::current_exe()?.parent() {
9272        directories.push(parent.to_path_buf());
9273    }
9274    directories.push(std::env::current_dir()?);
9275    if let Some(windows) = std::env::var_os("SystemRoot") {
9276        let windows = PathBuf::from(windows);
9277        directories.push(windows.join("System32"));
9278        directories.push(windows.clone());
9279    }
9280    let path = custom_path
9281        .map(|(_, value)| std::ffi::OsString::from(value))
9282        .or_else(|| std::env::var_os("PATH"));
9283    if let Some(path) = path {
9284        directories.extend(std::env::split_paths(&path));
9285    }
9286    for directory in directories {
9287        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9288            Ok(capture) => return Ok(capture),
9289            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9290            Err(error) => return Err(error),
9291        }
9292    }
9293    Err(io::Error::new(
9294        io::ErrorKind::NotFound,
9295        "resolved executable not found",
9296    ))
9297}
9298
9299fn spawn_and_mark_running(
9300    spec: &ModuleSpec,
9301    runtime: &SupervisorRuntimeConfig,
9302    snapshot: &SharedSnapshot,
9303) -> Result<SupervisedChild, SuperviseError> {
9304    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9305    let child = spawn_child(
9306        spec,
9307        runtime.connection_file_path.as_deref(),
9308        runtime.supervisor_handle.as_ref(),
9309        &runtime.stderr_ring,
9310        runtime.capture_logs_dir.as_deref(),
9311        &runtime.child_roster,
9312        #[cfg(target_os = "linux")]
9313        runtime.cgroup_placement.as_ref(),
9314    )?;
9315    set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9316    if let Some(spawn) = activity_spawn {
9317        spawn.complete(snapshot);
9318    }
9319    Ok(child)
9320}
9321
9322enum RegistrationWaitOutcome {
9323    Registered,
9324    Exited(ExitReport),
9325    TimedOut,
9326}
9327
9328struct ReloadRegistrationFailure {
9329    exit_report: ExitReport,
9330    reason: String,
9331}
9332
9333#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9334enum BusyGaugeObservation {
9335    Quiescent,
9336    Busy,
9337    Omitted,
9338}
9339
9340fn busy_gauge_observation(metrics: Option<&Value>, gauges: &[String]) -> BusyGaugeObservation {
9341    let Some(metrics) = metrics.and_then(Value::as_object) else {
9342        return BusyGaugeObservation::Omitted;
9343    };
9344    let mut sum = 0u128;
9345    for gauge in gauges {
9346        let Some(value) = metrics.get(gauge) else {
9347            return BusyGaugeObservation::Omitted;
9348        };
9349        let Some(value) = value.as_u64() else {
9350            return BusyGaugeObservation::Busy;
9351        };
9352        sum = sum.saturating_add(u128::from(value));
9353    }
9354    if sum == 0 {
9355        BusyGaugeObservation::Quiescent
9356    } else {
9357        BusyGaugeObservation::Busy
9358    }
9359}
9360
9361fn declared_busy_gauges(
9362    registry: &Registry,
9363    module_id: &str,
9364) -> Result<Vec<String>, SuperviseError> {
9365    busy_gauges_of(
9366        registry
9367            .get_module(module_id)
9368            .map_err(SuperviseError::Registry)?,
9369    )
9370}
9371
9372/// [`declared_busy_gauges`] for the registration a connection holds, in any
9373/// slot: after cutover the incumbent is no longer the id's active
9374/// registration, and its own manifest is the one that names its gauges.
9375fn declared_busy_gauges_for_connection(
9376    registry: &Registry,
9377    connection_id: ConnectionId,
9378) -> Result<Vec<String>, SuperviseError> {
9379    busy_gauges_of(
9380        registry
9381            .get_module_by_connection(connection_id)
9382            .map_err(SuperviseError::Registry)?,
9383    )
9384}
9385
9386fn busy_gauges_of(
9387    registration: Option<crate::registry::ModuleRegistration>,
9388) -> Result<Vec<String>, SuperviseError> {
9389    let Some(registration) = registration else {
9390        return Ok(Vec::new());
9391    };
9392    let Some(self_signals) = registration.manifest.self_signals else {
9393        return Ok(Vec::new());
9394    };
9395
9396    let mut gauges = Vec::new();
9397    for declaration in self_signals {
9398        if declaration.kind != SelfSignalKind::Busy {
9399            continue;
9400        }
9401        match declaration.anchored_to {
9402            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9403                gauges.extend(declared)
9404            }
9405            _ => {
9406                // An invalid Busy anchor is fail-safe: the empty name cannot be
9407                // present in a conforming health report, so this drain stays busy.
9408                gauges.push(String::new());
9409            }
9410        }
9411    }
9412    Ok(gauges)
9413}
9414
9415/// Wait for `endpoint` to have nothing in flight and, when the module declares
9416/// busy gauges, for a health probe to report them quiet. The probe is addressed
9417/// by `scope`: a swap's superseded incumbent must be asked about its own
9418/// gauges, and by module id the probe would reach the promoted candidate.
9419async fn wait_for_forwarding_quiescence(
9420    forwarding: &ForwardingTable,
9421    module_id: &str,
9422    runtime: &SupervisorRuntimeConfig,
9423    endpoint: crate::ModuleEndpointId,
9424    deadline: Instant,
9425    busy_gauges: &[String],
9426    scope: DrainScope,
9427) -> Result<bool, SuperviseError> {
9428    let mut gauges_quiescent = busy_gauges.is_empty();
9429    let mut next_probe_at = Instant::now();
9430    let mut omission_counted = false;
9431
9432    loop {
9433        let now = Instant::now();
9434        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9435            let report = match scope {
9436                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9437                DrainScope::Endpoint(endpoint) => {
9438                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9439                }
9440            };
9441            gauges_quiescent = match report {
9442                Ok(report) => match busy_gauge_observation(report.metrics.as_ref(), busy_gauges) {
9443                    BusyGaugeObservation::Quiescent => true,
9444                    BusyGaugeObservation::Busy => false,
9445                    BusyGaugeObservation::Omitted => {
9446                        if !omission_counted {
9447                            forwarding
9448                                .counters()
9449                                .increment_drains_with_undeclared_gauge();
9450                            omission_counted = true;
9451                        }
9452                        false
9453                    }
9454                },
9455                Err(err) => {
9456                    warn!(
9457                        module_id,
9458                        error = %err,
9459                        "drain health.check did not produce declared busy gauges; treating module as busy"
9460                    );
9461                    false
9462                }
9463            };
9464            next_probe_at = Instant::now() + runtime.health.cadence.max(REGISTRY_RELEASE_POLL);
9465        }
9466
9467        let in_flight = forwarding
9468            .endpoint_in_flight_count(endpoint)
9469            .map_err(SuperviseError::Forwarding)?;
9470        if in_flight == 0 && gauges_quiescent {
9471            return Ok(true);
9472        }
9473
9474        let now = Instant::now();
9475        if now >= deadline {
9476            return Ok(false);
9477        }
9478        let mut wait = deadline
9479            .saturating_duration_since(now)
9480            .min(REGISTRY_RELEASE_POLL);
9481        if !busy_gauges.is_empty() {
9482            wait = wait.min(next_probe_at.saturating_duration_since(now));
9483        }
9484        sleep(wait).await;
9485    }
9486}
9487
9488/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
9489///
9490/// `Ok` is always honest and passed straight through -- the wait actually measured
9491/// in-flight state. `Err` means the wait produced no measurement at all (the
9492/// forwarding table's lock was poisoned), so `false` is reported as the one honest
9493/// constant: the drain did not complete. Never recomputed from route state, never a
9494/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
9495fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
9496    match wait_result {
9497        Ok(drained) => *drained,
9498        Err(_) => false,
9499    }
9500}
9501
9502fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
9503    for released in released_routes {
9504        let frame = match Frame::build_with_version(
9505            released.negotiated_ver,
9506            FrameType::Goodbye,
9507            control_flags(),
9508            released.channel,
9509            released.epoch,
9510            0,
9511            Vec::new(),
9512        ) {
9513            Ok(frame) => frame,
9514            Err(err) => {
9515                warn!(
9516                    route_channel = released.channel,
9517                    error = %err,
9518                    "failed to build supervisor drain route GOODBYE frame"
9519                );
9520                continue;
9521            }
9522        };
9523        if !released.close_on_delivery_failure() {
9524            crate::forwarding::send_module_route_goodbye(
9525                &forwarding.counters(),
9526                &released.sink,
9527                frame,
9528                released.module_id.as_deref(),
9529                "supervisor drain",
9530            );
9531            continue;
9532        }
9533        if let Err(err) = released.sink.try_send(frame) {
9534            warn!(
9535                target_connection_id = released.connection_id.get(),
9536                route_channel = released.channel,
9537                error = %err,
9538                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
9539            );
9540            let _ = forwarding.escalate_client_delivery_failure(
9541                released.connection_id,
9542                released.channel,
9543                released.epoch,
9544                CloseReason::new(
9545                    "route_goodbye_delivery_failed",
9546                    format!(
9547                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
9548                        released.channel
9549                    ),
9550                ),
9551                crate::forwarding::UndeliveredFrame {
9552                    module_id: released.module_id.as_deref(),
9553                    sink: &released.sink,
9554                },
9555            );
9556        }
9557    }
9558}
9559
9560fn send_module_draining(
9561    module_id: &str,
9562    reason: RouteCloseReason,
9563    deadline_ms: u64,
9564    target: &ModuleDrainTarget,
9565) {
9566    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
9567        reason,
9568        deadline_ms,
9569    }) {
9570        Ok(body) => body,
9571        Err(err) => {
9572            warn!(
9573                module_id,
9574                error = %err,
9575                "failed to encode module draining command"
9576            );
9577            return;
9578        }
9579    };
9580    let frame = match Frame::build_with_version(
9581        target.negotiated_ver,
9582        FrameType::Push,
9583        control_flags(),
9584        0,
9585        0,
9586        0,
9587        body,
9588    ) {
9589        Ok(frame) => frame,
9590        Err(err) => {
9591            warn!(
9592                module_id,
9593                error = %err,
9594                "failed to build module draining command frame"
9595            );
9596            return;
9597        }
9598    };
9599    if let Err(err) = target.sink.try_send(frame) {
9600        warn!(
9601            module_id,
9602            target_connection_id = target.endpoint.connection_id.get(),
9603            error = %err,
9604            "module draining command was not delivered to peer"
9605        );
9606    }
9607}
9608
9609/// The channel-0 GOODBYE that tells a module its stop is planned.
9610fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
9611    match Frame::build_with_version(
9612        negotiated_ver,
9613        FrameType::Goodbye,
9614        control_flags(),
9615        0,
9616        0,
9617        0,
9618        Vec::new(),
9619    ) {
9620        Ok(frame) => Some(frame),
9621        Err(err) => {
9622            warn!(
9623                module_id,
9624                error = %err,
9625                "failed to build module GOODBYE frame"
9626            );
9627            None
9628        }
9629    }
9630}
9631
9632/// Send every registered module connection its module GOODBYE at daemon
9633/// shutdown, then request that connection's close.
9634///
9635/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
9636/// before EOF, so the GOODBYE must reach the socket before the close. A close
9637/// request does not wait for the connection's queued frames: its writer gets a
9638/// bounded grace after the close, is aborted if it overruns it, and the daemon
9639/// process may exit before that grace ends. So with `wait_for_flush`, each
9640/// connection is closed only after its writer has acknowledged writing the
9641/// GOODBYE, or once a short shared budget runs out, so one module that is not
9642/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
9643/// are only queued, for a shutdown the operator has told to stop waiting.
9644/// A connection that is already gone is skipped.
9645#[cfg(unix)]
9646async fn send_module_goodbyes_for_daemon_shutdown(
9647    forwarding: &Arc<ForwardingTable>,
9648    reason: &CloseReason,
9649    wait_for_flush: bool,
9650) {
9651    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
9652    let targets = match forwarding.module_connections() {
9653        Ok(targets) => targets,
9654        Err(err) => {
9655            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
9656            return;
9657        }
9658    };
9659    let deadline = Instant::now() + GOODBYE_BUDGET;
9660    let mut sends = tokio::task::JoinSet::new();
9661    for target in targets {
9662        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
9663            continue;
9664        };
9665        if !wait_for_flush {
9666            if let Err(err) = target.sink.try_send(frame) {
9667                debug!(
9668                    module_id = %target.module_id,
9669                    error = %err,
9670                    "shutdown module GOODBYE was not queued"
9671                );
9672            }
9673            continue;
9674        }
9675        let forwarding = Arc::clone(forwarding);
9676        let reason = reason.clone();
9677        sends.spawn(async move {
9678            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
9679                Ok(Ok(())) => {}
9680                Ok(Err(err)) => debug!(
9681                    module_id = %target.module_id,
9682                    error = %err,
9683                    "module connection closed before its shutdown GOODBYE was written"
9684                ),
9685                Err(_) => warn!(
9686                    module_id = %target.module_id,
9687                    budget = ?GOODBYE_BUDGET,
9688                    "shutdown module GOODBYE was not written within its budget; closing anyway"
9689                ),
9690            }
9691            forwarding.request_connection_close(target.endpoint.connection_id, reason);
9692        });
9693    }
9694    // Every task ends by the shared deadline, so this wait is bounded too.
9695    while sends.join_next().await.is_some() {}
9696}
9697
9698fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
9699    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
9700        return;
9701    };
9702    if let Err(err) = target.sink.try_send(frame) {
9703        warn!(
9704            module_id,
9705            target_connection_id = target.endpoint.connection_id.get(),
9706            error = %err,
9707            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
9708        );
9709        forwarding.request_connection_close(
9710            target.endpoint.connection_id,
9711            CloseReason::new(
9712                "module_goodbye_delivery_failed",
9713                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
9714            ),
9715        );
9716    }
9717}
9718
9719#[derive(Clone, Copy)]
9720struct ForwardingDrainContext<'a> {
9721    spec: &'a ModuleSpec,
9722    runtime: &'a SupervisorRuntimeConfig,
9723    registry: &'a Registry,
9724    scope: DrainScope,
9725}
9726
9727/// Which process a forwarding drain addresses.
9728#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9729enum DrainScope {
9730    /// Whatever endpoint is active for the module id: every plain stop,
9731    /// restart and reload. Also moves the module's state to `Draining`.
9732    Active,
9733    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
9734    /// module id would resolve to the promoted candidate and leave neither
9735    /// process routable. The module's state is left alone, since the promoted
9736    /// candidate is what it describes and that process is running.
9737    Endpoint(crate::ModuleEndpointId),
9738}
9739
9740/// Whether a child being drained has already been asked to stop by the time
9741/// its drain wait starts.
9742///
9743/// The drain wait is the same budget whatever this says. What it decides is
9744/// whether the supervisor must ask by signal before that wait begins: a child
9745/// that nobody asked will sit out the whole budget and then be SIGKILLed,
9746/// healthy or not.
9747#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9748enum StopNotice {
9749    /// The module was sent `module.draining` and a module GOODBYE over its own
9750    /// registered connection, and stops itself.
9751    SentOverConnection,
9752    /// The forwarding drain found no registered connection for the module: a
9753    /// subc child spawned moments ago that has not sent HELLO yet, or a
9754    /// `protocol: "none"` child, which never registers.
9755    NoConnection,
9756    /// This path sends nothing over the module's connection: the supervisor has
9757    /// no forwarding table, or the caller stops the child without a forwarding
9758    /// drain.
9759    NotSent,
9760}
9761
9762async fn begin_forwarding_drain(
9763    spec: &ModuleSpec,
9764    runtime: &SupervisorRuntimeConfig,
9765    registry: &Registry,
9766    snapshot: &SharedSnapshot,
9767    enabled: Option<bool>,
9768    reason: RouteCloseReason,
9769) -> Result<StopNotice, SuperviseError> {
9770    let Some(forwarding) = runtime.forwarding.as_ref() else {
9771        return Err(SuperviseError::ReloadUnavailable {
9772            module_id: spec.module_id.clone(),
9773            reason: "supervisor was not configured with a forwarding table".to_string(),
9774        });
9775    };
9776
9777    begin_forwarding_drain_with(
9778        forwarding,
9779        ForwardingDrainContext {
9780            spec,
9781            runtime,
9782            registry,
9783            scope: DrainScope::Active,
9784        },
9785        snapshot,
9786        enabled,
9787        reason,
9788        runtime.drain_timeout,
9789    )
9790    .await
9791}
9792
9793async fn begin_forwarding_drain_if_configured(
9794    spec: &ModuleSpec,
9795    runtime: &SupervisorRuntimeConfig,
9796    registry: &Registry,
9797    snapshot: &SharedSnapshot,
9798    enabled: Option<bool>,
9799    reason: RouteCloseReason,
9800) -> Result<StopNotice, SuperviseError> {
9801    begin_forwarding_drain_with_timeout(
9802        spec,
9803        runtime,
9804        registry,
9805        snapshot,
9806        enabled,
9807        reason,
9808        runtime.drain_timeout,
9809    )
9810    .await
9811}
9812
9813/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
9814/// budget, for paths where the operator overrides the module's configured one
9815/// (`supervisor.restart{drain_timeout_ms}`).
9816async fn begin_forwarding_drain_with_timeout(
9817    spec: &ModuleSpec,
9818    runtime: &SupervisorRuntimeConfig,
9819    registry: &Registry,
9820    snapshot: &SharedSnapshot,
9821    enabled: Option<bool>,
9822    reason: RouteCloseReason,
9823    drain_timeout: Duration,
9824) -> Result<StopNotice, SuperviseError> {
9825    let Some(forwarding) = runtime.forwarding.as_ref() else {
9826        return Ok(StopNotice::NotSent);
9827    };
9828
9829    begin_forwarding_drain_with(
9830        forwarding,
9831        ForwardingDrainContext {
9832            spec,
9833            runtime,
9834            registry,
9835            scope: DrainScope::Active,
9836        },
9837        snapshot,
9838        enabled,
9839        reason,
9840        drain_timeout,
9841    )
9842    .await
9843}
9844
9845async fn begin_forwarding_drain_with(
9846    forwarding: &ForwardingTable,
9847    context: ForwardingDrainContext<'_>,
9848    snapshot: &SharedSnapshot,
9849    enabled: Option<bool>,
9850    reason: RouteCloseReason,
9851    drain_timeout: Duration,
9852) -> Result<StopNotice, SuperviseError> {
9853    let ForwardingDrainContext {
9854        spec,
9855        runtime,
9856        registry,
9857        scope,
9858    } = context;
9859    debug_assert_ne!(reason, RouteCloseReason::Crash);
9860    let terminal = matches!(reason, RouteCloseReason::Disable);
9861    let drain_started_at = Instant::now();
9862    let drain_deadline = drain_started_at + drain_timeout;
9863    let deadline_ms =
9864        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
9865    let busy_gauges = match scope {
9866        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
9867        DrainScope::Endpoint(endpoint) => {
9868            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
9869        }
9870    };
9871
9872    // Admission gate first: route.open/commit and route REQUEST admission are closed
9873    // before the first quiescence check, so the outstanding count can only fall.
9874    let gate_started = Instant::now();
9875    let drain_target = match scope {
9876        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
9877        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
9878    }
9879    .map_err(SuperviseError::Forwarding)?;
9880    // The instant admission closed, and how long taking the forwarding write
9881    // lock to close it took. The timeout line reports only the quiescence
9882    // wait, so without this a drain that started late looked like one that
9883    // started on time.
9884    info!(
9885        module_id = %spec.module_id,
9886        ?reason,
9887        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
9888        connected = drain_target.is_some(),
9889        "module drain began; route admission closed"
9890    );
9891    if scope == DrainScope::Active {
9892        update_snapshot(snapshot, Some(&spec.module_id), |state| {
9893            state.state = ModuleState::Draining;
9894            state.draining_to_replace =
9895                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
9896            if let Some(enabled) = enabled {
9897                state.enabled = enabled;
9898            }
9899        })?;
9900    }
9901
9902    let Some(target) = drain_target.as_ref() else {
9903        // Nothing was sent: the module has no registered connection to carry
9904        // `module.draining` or a GOODBYE. The caller must not assume the child
9905        // was asked to stop.
9906        return Ok(StopNotice::NoConnection);
9907    };
9908    {
9909        send_module_draining(&spec.module_id, reason, deadline_ms, target);
9910        #[cfg(feature = "test-support")]
9911        forwarding.wait_before_route_closing_for_test().await;
9912        let routes = forwarding
9913            .endpoint_routes(target.endpoint)
9914            .map_err(SuperviseError::Forwarding)?;
9915        let routes_notified = routes.len();
9916        crate::control::send_route_control_pushes(
9917            forwarding,
9918            routes.clone(),
9919            ClientControlPush::RouteClosing {
9920                module_id: spec.module_id.clone(),
9921                channels: Vec::new(),
9922                reason,
9923            },
9924        );
9925        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
9926
9927        // `route.closing` was just sent above: from here on every return path,
9928        // including an early one, MUST send `route.closed` before propagating
9929        // anything else. A client holds `closing` as a promise that a verdict is
9930        // coming; leaving early without `closed` strands it waiting forever, since
9931        // `closing` carries no timeout of its own.
9932        let wait_result = wait_for_forwarding_quiescence(
9933            forwarding,
9934            &spec.module_id,
9935            runtime,
9936            target.endpoint,
9937            drain_deadline,
9938            &busy_gauges,
9939            scope,
9940        )
9941        .await;
9942        let drained = drained_after_quiescence_wait(&wait_result);
9943        if let Err(err) = &wait_result {
9944            error!(
9945                module_id = %spec.module_id,
9946                ?reason,
9947                error = %err,
9948                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
9949            );
9950        } else if !drained {
9951            // Name what the drain waited on. Without it the line says only that
9952            // something did not settle, and "one wedged call" and "every
9953            // session's held stream" read the same; the first is a module bug,
9954            // the second is a module that should end its streams on
9955            // module.draining. Read before teardown releases the routes.
9956            let holdouts = forwarding
9957                .endpoint_drain_holdouts(target.endpoint)
9958                .unwrap_or_default();
9959            warn!(
9960                module_id = %spec.module_id,
9961                waited = ?drain_timeout,
9962                ?reason,
9963                held_requests = holdouts.requests,
9964                held_routes = holdouts.routes,
9965                total_routes = holdouts.total_routes,
9966                top_connections = ?holdouts.top_connections,
9967                // `module_channel:corr`, so the module can find each held request
9968                // in its own log; capped, so `held_requests` is the full count.
9969                held = %holdouts
9970                    .held
9971                    .iter()
9972                    .map(|(channel, corr)| format!("{channel}:{corr}"))
9973                    .collect::<Vec<_>>()
9974                    .join(","),
9975                "route drain timed out before request quiescence; forcing teardown"
9976            );
9977        }
9978        crate::control::send_route_control_pushes(
9979            forwarding,
9980            routes,
9981            ClientControlPush::RouteClosed {
9982                module_id: spec.module_id.clone(),
9983                channels: Vec::new(),
9984                reason,
9985                drained,
9986                abandoned: target.abandoned_bindings.len() as u32,
9987                excluded_subscriptions: target.excluded_subscriptions,
9988                terminal: Some(terminal),
9989            },
9990        );
9991        wait_result?;
9992
9993        // `route.closed` has now been sent unconditionally above. From here the
9994        // remaining steps are cleanup (route + module GOODBYE) rather than a
9995        // promise the client is waiting on, but a lock-poisoned
9996        // `release_module_endpoint_routes` would otherwise skip the module
9997        // GOODBYE silently too -- send it before propagating the error.
9998        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
9999            Ok(routes) => routes,
10000            Err(err) => {
10001                warn!(
10002                    module_id = %spec.module_id,
10003                    ?reason,
10004                    error = %err,
10005                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10006                );
10007                send_module_goodbye(&spec.module_id, forwarding, target);
10008                return Err(SuperviseError::Forwarding(err));
10009            }
10010        };
10011        let route_goodbye_count = released_routes.len();
10012        send_route_goodbyes(forwarding, released_routes);
10013        send_module_goodbye(&spec.module_id, forwarding, target);
10014
10015        // The drain's happy path was previously silent: every emission above is
10016        // best-effort with only its failure arm logged, so "were consumers told"
10017        // was unprovable from the daemon log (surfaced by a 30-minute consumer
10018        // hang where the open question was exactly whether teardown notice went
10019        // out). One summary line makes that class decidable in one grep.
10020        info!(
10021            module_id = %spec.module_id,
10022            ?reason,
10023            routes_notified,
10024            route_goodbyes = route_goodbye_count,
10025            abandoned_reservations = target.abandoned_bindings.len(),
10026            excluded_subscriptions = target.excluded_subscriptions,
10027            drained,
10028            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10029        );
10030    }
10031
10032    Ok(StopNotice::SentOverConnection)
10033}
10034
10035/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
10036/// the only slot a plain (non-swap) spawn can register into.
10037async fn wait_for_registration_after_reload(
10038    registry: &Registry,
10039    module_id: &str,
10040    snapshot: &SharedSnapshot,
10041    child: &mut SupervisedChild,
10042    wait: Duration,
10043) -> Result<RegistrationWaitOutcome, SuperviseError> {
10044    wait_for_slot_registration(
10045        registry,
10046        crate::registry::RegistrationSlot::Active(module_id),
10047        module_id,
10048        snapshot,
10049        child,
10050        wait,
10051    )
10052    .await
10053}
10054
10055/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
10056///
10057/// Keyed on the slot rather than the bare module id because during a swap the
10058/// id's active slot is already held by the incumbent: an id-keyed wait would
10059/// report the incumbent's registration as the candidate's and a candidate that
10060/// never registers would look registered. A swap candidate waits on
10061/// `crate::registry::RegistrationSlot::Candidate`.
10062async fn wait_for_slot_registration(
10063    registry: &Registry,
10064    slot: crate::registry::RegistrationSlot<'_>,
10065    module_id: &str,
10066    snapshot: &SharedSnapshot,
10067    child: &mut SupervisedChild,
10068    wait: Duration,
10069) -> Result<RegistrationWaitOutcome, SuperviseError> {
10070    let deadline = Instant::now() + wait;
10071    loop {
10072        if registry
10073            .registration(slot)
10074            .map_err(SuperviseError::Registry)?
10075            .is_some()
10076        {
10077            return Ok(RegistrationWaitOutcome::Registered);
10078        }
10079
10080        let now = Instant::now();
10081        if now >= deadline {
10082            return Ok(RegistrationWaitOutcome::TimedOut);
10083        }
10084        let remaining = deadline.saturating_duration_since(now);
10085        let poll = remaining.min(REGISTRY_RELEASE_POLL);
10086
10087        tokio::select! {
10088            wait_result = child.wait() => {
10089                let status = wait_result.map_err(|source| SuperviseError::Wait {
10090                    module_id: module_id.to_string(),
10091                    source,
10092                })?;
10093                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10094                    snapshot,
10095                    child,
10096                    &status,
10097                )));
10098            }
10099            _ = sleep(poll) => {}
10100        }
10101    }
10102}
10103
10104fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10105    // A replacement process that exits before HELLO did not provide service, even
10106    // if it used status 0. Count it against the restart cap as a new-binary failure.
10107    if exit_report.kind != ExitKind::DeliberateSeverance {
10108        exit_report.kind = ExitKind::Crash;
10109    }
10110    exit_report
10111}
10112
10113async fn handle_reload_child_registration_failure(
10114    spec: &ModuleSpec,
10115    runtime: &SupervisorRuntimeConfig,
10116    registry: &Registry,
10117    process_liveness: &SupervisorProcessLiveness,
10118    snapshot: &SharedSnapshot,
10119    _child: &mut Option<SupervisedChild>,
10120    failure: ReloadRegistrationFailure,
10121) -> Result<(), SuperviseError> {
10122    let ReloadRegistrationFailure {
10123        exit_report,
10124        reason,
10125    } = failure;
10126    match on_child_exit(
10127        spec,
10128        runtime.restart_policy,
10129        registry,
10130        snapshot,
10131        &runtime.terminal_ring,
10132        &runtime.spawn_events,
10133        &runtime.child_roster,
10134        exit_report,
10135    )
10136    .await
10137    {
10138        NextAction::Stop {
10139            registration_released,
10140        } => {
10141            if registration_released {
10142                process_liveness.untrack_if_current(&spec.module_id, snapshot);
10143            }
10144        }
10145        NextAction::Restart { schedule } => {
10146            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10147                schedule.delay
10148            });
10149            if let Some(schedule) = schedule {
10150                log_crash_respawn(&spec.module_id, schedule);
10151            }
10152            schedule_respawn(
10153                runtime,
10154                snapshot,
10155                &spec.module_id,
10156                delay,
10157                RespawnKind::Spawn,
10158            )?;
10159        }
10160    }
10161    Err(SuperviseError::ReloadFailed {
10162        module_id: spec.module_id.clone(),
10163        reason,
10164    })
10165}
10166
10167async fn handle_reload_spawn_failure(
10168    spec: &ModuleSpec,
10169    runtime: &SupervisorRuntimeConfig,
10170    process_liveness: &SupervisorProcessLiveness,
10171    snapshot: &SharedSnapshot,
10172    _child: &mut Option<SupervisedChild>,
10173    reason: String,
10174) -> Result<(), SuperviseError> {
10175    let now = Instant::now();
10176    let mut schedule = None;
10177    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10178        clear_current_process_facts(state);
10179        if state.enabled {
10180            schedule = state.next_crash_restart(&runtime.restart_policy, now);
10181            state.state = if schedule.is_some() {
10182                ModuleState::Restarting
10183            } else {
10184                ModuleState::Failed
10185            };
10186        } else {
10187            state.state = ModuleState::Disabled;
10188        }
10189    })?;
10190    if let Some(schedule) = schedule {
10191        schedule_respawn(
10192            runtime,
10193            snapshot,
10194            &spec.module_id,
10195            schedule.delay,
10196            RespawnKind::Spawn,
10197        )?;
10198    } else {
10199        process_liveness.untrack_if_current(&spec.module_id, snapshot);
10200    }
10201    Err(SuperviseError::ReloadFailed {
10202        module_id: spec.module_id.clone(),
10203        reason,
10204    })
10205}
10206
10207fn control_flags() -> Flags {
10208    Flags::new(false, Priority::Passive, false)
10209}
10210
10211#[allow(clippy::too_many_arguments)]
10212async fn drain_optional_child(
10213    module_id: &str,
10214    protocol: ModuleProtocol,
10215    stop_notice: StopNotice,
10216    registry: &Registry,
10217    forwarding: Option<&ForwardingTable>,
10218    snapshot: &SharedSnapshot,
10219    terminal_ring: &Arc<Mutex<TerminalRing>>,
10220    spawn_events: &SpawnEventFeed,
10221    child: &mut Option<SupervisedChild>,
10222    drain_timeout: Duration,
10223    final_state: ModuleState,
10224    enabled: Option<bool>,
10225) -> Result<(), SuperviseError> {
10226    if let Some(child) = child.take() {
10227        drain_child_to_state(
10228            module_id,
10229            protocol,
10230            stop_notice,
10231            registry,
10232            forwarding,
10233            snapshot,
10234            terminal_ring,
10235            spawn_events,
10236            child,
10237            drain_timeout,
10238            final_state,
10239            enabled,
10240        )
10241        .await
10242    } else {
10243        update_snapshot(snapshot, Some(module_id), |state| {
10244            state.state = final_state;
10245            if let Some(enabled) = enabled {
10246                state.enabled = enabled;
10247            }
10248            clear_current_process_facts(state);
10249        })?;
10250        release_dead_registration(registry, forwarding, snapshot, module_id).await
10251    }
10252}
10253
10254#[allow(clippy::too_many_arguments)]
10255async fn drain_child_to_state(
10256    module_id: &str,
10257    _protocol: ModuleProtocol,
10258    stop_notice: StopNotice,
10259    registry: &Registry,
10260    forwarding: Option<&ForwardingTable>,
10261    snapshot: &SharedSnapshot,
10262    terminal_ring: &Arc<Mutex<TerminalRing>>,
10263    spawn_events: &SpawnEventFeed,
10264    mut child: SupervisedChild,
10265    drain_timeout: Duration,
10266    final_state: ModuleState,
10267    enabled: Option<bool>,
10268) -> Result<(), SuperviseError> {
10269    let protocol = child.protocol;
10270    update_snapshot(snapshot, Some(module_id), |state| {
10271        state.state = ModuleState::Draining;
10272        state.draining_to_replace = final_state == ModuleState::Restarting;
10273        if let Some(enabled) = enabled {
10274            state.enabled = enabled;
10275        }
10276    })?;
10277
10278    // The wait below is the same budget in every case; what differs is
10279    // whether anything has ASKED the child to stop before it starts. Only a
10280    // forwarding drain that reached the module's registered connection has
10281    // (`module.draining`, then a module GOODBYE). Every other child was told
10282    // nothing: a `protocol: "none"` module, which never registers; a subc
10283    // module spawned moments ago that has not sent HELLO yet; or a stop that
10284    // runs no forwarding drain. Without a signal the budget is only a delay
10285    // in front of SIGKILL -- and the not-yet-registered child is the worst
10286    // case, because it registers into a module that is already draining,
10287    // is never told, and is killed while healthy.
10288    if stop_notice != StopNotice::SentOverConnection {
10289        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10290            info!(
10291                module_id,
10292                pid = child.pid,
10293                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10294                "module has no connection yet; requesting stop by signal"
10295            );
10296        }
10297        request_graceful_stop(module_id, &child);
10298    }
10299
10300    let exit_report = match timeout(drain_timeout, child.wait()).await {
10301        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10302        Ok(Err(source)) => {
10303            fail_after_wait_error(
10304                module_id,
10305                snapshot,
10306                terminal_ring,
10307                spawn_events,
10308                format!("failed to wait during drain: {source}"),
10309            );
10310            return Err(SuperviseError::Wait {
10311                module_id: module_id.to_string(),
10312                source,
10313            });
10314        }
10315        Err(_) => {
10316            // Mirror the sibling arm above: state is already `Draining`, and an
10317            // error propagated from here would strand it there -- a state
10318            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
10319            // `Failed | Stopped`), leaving an operator Restart as the only exit.
10320            // `Failed` before `?` keeps the module operator-visible and
10321            // revivable. Trigger is an ESRCH race (process exits between the
10322            // drain timeout firing and the kill) or a post-kill wait failure
10323            // (issue #34).
10324            //
10325            // Logged because the kill is otherwise visible only as signal 9 in
10326            // the terminal ring, and the budget it follows can be long enough
10327            // that consumers see a stretch of refusals with no stated cause.
10328            warn!(
10329                module_id,
10330                pid = child.pid,
10331                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10332                reason = ?final_state,
10333                ?stop_notice,
10334                "drain budget expired before the module exited; killing it"
10335            );
10336            child.start_kill().map_err(|source| {
10337                fail_after_wait_error(
10338                    module_id,
10339                    snapshot,
10340                    terminal_ring,
10341                    spawn_events,
10342                    format!("failed to kill during drain: {source}"),
10343                );
10344                SuperviseError::Kill {
10345                    module_id: module_id.to_string(),
10346                    source,
10347                }
10348            })?;
10349            let status = child.wait().await.map_err(|source| {
10350                fail_after_wait_error(
10351                    module_id,
10352                    snapshot,
10353                    terminal_ring,
10354                    spawn_events,
10355                    format!("failed to wait after drain kill: {source}"),
10356                );
10357                SuperviseError::Wait {
10358                    module_id: module_id.to_string(),
10359                    source,
10360                }
10361            })?;
10362            classify_reaped_child_exit(snapshot, &child, &status)
10363        }
10364    };
10365
10366    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10367    publish_after_terminal_record(
10368        module_id,
10369        || {
10370            record_terminal_with_detail(
10371                module_id,
10372                terminal_ring,
10373                spawn_events,
10374                &exit_report,
10375                terminal_disposition(final_state),
10376                detail,
10377            )
10378        },
10379        || {
10380            update_snapshot(snapshot, Some(module_id), |state| {
10381                state.state = final_state;
10382                if let Some(enabled) = enabled {
10383                    state.enabled = enabled;
10384                }
10385                clear_current_process_facts(state);
10386                state.last_exit = Some(exit_report.clone());
10387                if exit_report.kind == ExitKind::DeliberateSeverance {
10388                    state.lifetime_restarts += 1;
10389                }
10390            })
10391        },
10392    )?;
10393    child.drain_stderr(module_id).await;
10394
10395    release_dead_registration(registry, forwarding, snapshot, module_id).await
10396}
10397
10398/// Ask a child that nothing else has asked to stop, by signal.
10399///
10400/// A registered subc module is asked over its own connection: the drain sends
10401/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
10402/// module GOODBYE, and the module stops itself. A module that speaks no subc
10403/// wire receives none of that, and neither does a subc module that has not
10404/// registered yet, so for them the drain budget would be pure delay in front of
10405/// a SIGKILL -- and for a process with a store to flush (JetStream is the
10406/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
10407/// into a recovery on the next start.
10408///
10409/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
10410/// rule rather than an optimisation: that module's graceful stop is already
10411/// running by the time its child is drained, and a signal would race it.
10412///
10413/// Best-effort by construction. A child that has already exited is the ordinary
10414/// case rather than an error (the kill lands on a reaped or exiting pid), so a
10415/// failure is logged at debug and the wait-then-kill below still decides the
10416/// outcome.
10417#[cfg(unix)]
10418fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10419    let Some(pid) = child
10420        .id()
10421        .and_then(|pid| i32::try_from(pid).ok())
10422        .and_then(rustix::process::Pid::from_raw)
10423    else {
10424        debug!(
10425            module_id,
10426            "no pid to signal for teardown; falling through to the drain wait"
10427        );
10428        return;
10429    };
10430    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10431        Ok(()) => debug!(
10432            module_id,
10433            "sent SIGTERM to a module nothing else asked to stop"
10434        ),
10435        Err(err) => debug!(
10436            module_id,
10437            error = %err,
10438            "SIGTERM to module failed; the drain wait and kill still apply"
10439        ),
10440    }
10441}
10442
10443/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
10444/// Windows does offer need cooperation this supervisor cannot assume: a console
10445/// control event requires sharing a console with the child, and `WM_CLOSE`
10446/// requires the child to pump a message loop. A supervised server process does
10447/// neither, so there is nothing to send and teardown is the wait followed by the
10448/// kill. Emulating a signal here would mean inventing a stop protocol, which is
10449/// the thing `protocol: "none"` exists to avoid.
10450#[cfg(not(unix))]
10451fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
10452    debug!(
10453        module_id,
10454        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
10455    );
10456}
10457
10458fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
10459    match final_state {
10460        ModuleState::Stopped => TerminalDisposition::Stopped,
10461        ModuleState::Disabled => TerminalDisposition::Disabled,
10462        ModuleState::Restarting => TerminalDisposition::Restarting,
10463        ModuleState::Failed => TerminalDisposition::Failed,
10464        ModuleState::Starting
10465        | ModuleState::Running
10466        | ModuleState::Unresponsive
10467        | ModuleState::Draining => {
10468            unreachable!("terminal exits only finish in terminal or restarting states")
10469        }
10470    }
10471}
10472
10473/// Release a reaped child's registration before allowing another spawn.
10474///
10475/// EOF is not a process-lifetime signal: an inherited socket can stay open
10476/// indefinitely, and serial frame dispatch can be waiting on egress instead of
10477/// reading EOF. After the normal release grace, request connection close (which
10478/// cancels both reads and dispatch), then allow one more release grace for the
10479/// connection guard's forwarding cleanup. Never evict a different connection.
10480async fn release_dead_registration(
10481    registry: &Registry,
10482    forwarding: Option<&ForwardingTable>,
10483    snapshot: &SharedSnapshot,
10484    module_id: &str,
10485) -> Result<(), SuperviseError> {
10486    let result = async {
10487        let registration = registry
10488            .get_module(module_id)
10489            .map_err(SuperviseError::Registry)?;
10490        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
10491            Ok(()) => return Ok(()),
10492            Err(SuperviseError::RegistrationStillActive { .. }) => {}
10493            Err(err) => return Err(err),
10494        }
10495        let pid = lock_snapshot(snapshot)?.reaped_pid;
10496        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
10497            warn!(
10498                module_id,
10499                pid,
10500                connection_id = registration.connection_id.get(),
10501                "reaped module registration outlived release grace; closing dead connection"
10502            );
10503            forwarding.request_connection_close(
10504                registration.connection_id,
10505                CloseReason::new(
10506                    "supervised_process_reaped",
10507                    format!("module '{module_id}' pid {pid} exited"),
10508                ),
10509            );
10510            wait_for_slot_registration_release(
10511                registry,
10512                crate::registry::RegistrationSlot::Connection(registration.connection_id),
10513                REGISTRY_RELEASE_TIMEOUT,
10514            )
10515            .await?;
10516        }
10517        wait_for_registration_release(registry, module_id, Duration::ZERO).await
10518    }
10519    .await;
10520    if let Err(err) = &result {
10521        fail_snapshot(snapshot, Some(module_id), None);
10522        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
10523    }
10524    result
10525}
10526
10527/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
10528/// plain stop or restart waits for before it spawns a replacement.
10529async fn wait_for_registration_release(
10530    registry: &Registry,
10531    module_id: &str,
10532    wait: Duration,
10533) -> Result<(), SuperviseError> {
10534    wait_for_slot_registration_release(
10535        registry,
10536        crate::registry::RegistrationSlot::Active(module_id),
10537        wait,
10538    )
10539    .await
10540}
10541
10542/// Wait for the registration in `slot` to go away.
10543///
10544/// Keyed on the slot rather than the bare module id because a successful swap
10545/// never empties the id's active slot (the promoted candidate is in it), so an
10546/// id-keyed wait for the incumbent's release would always time out. Draining a
10547/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
10548/// incumbent's connection instead.
10549async fn wait_for_slot_registration_release(
10550    registry: &Registry,
10551    slot: crate::registry::RegistrationSlot<'_>,
10552    wait: Duration,
10553) -> Result<(), SuperviseError> {
10554    let deadline = Instant::now() + wait;
10555    let mut release_events = registration_release_events().subscribe();
10556    let still_active = |registration: &crate::registry::ModuleRegistration| {
10557        SuperviseError::RegistrationStillActive {
10558            module_id: registration.manifest.module_id.clone(),
10559            waited: wait,
10560        }
10561    };
10562    loop {
10563        let _observed_generation = *release_events.borrow_and_update();
10564        let Some(registration) = registry
10565            .registration(slot)
10566            .map_err(SuperviseError::Registry)?
10567        else {
10568            return Ok(());
10569        };
10570
10571        let now = Instant::now();
10572        if now >= deadline {
10573            return Err(still_active(&registration));
10574        }
10575
10576        let remaining = deadline.saturating_duration_since(now);
10577        match timeout(remaining, release_events.changed()).await {
10578            Ok(Ok(())) | Ok(Err(_)) => {}
10579            Err(_) => return Err(still_active(&registration)),
10580        }
10581    }
10582}
10583
10584#[cfg(test)]
10585mod slot_registration_wait_tests {
10586    use super::*;
10587    use crate::registry::{ConnectionId, RegistrationSlot};
10588    use subc_protocol::manifest::ModuleManifest;
10589
10590    #[tokio::test]
10591    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
10592        let registry = Arc::new(Registry::default());
10593        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
10594        let runtime = supervisor.runtime_config();
10595        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
10596        let spec = ModuleSpec {
10597            module_id: "enable-stale-registration".to_string(),
10598            program: PathBuf::from("/missing/enable-retry-test"),
10599            args: Vec::new(),
10600            env: Vec::new(),
10601            reserved: false,
10602            reserved_prefixes: Vec::new(),
10603            protocol: ModuleProtocol::Subc,
10604            overlap: Default::default(),
10605        };
10606        let connection = ConnectionId::new(90);
10607        registry
10608            .register_with_control_ops(
10609                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
10610                1,
10611                connection,
10612                Vec::new(),
10613            )
10614            .unwrap();
10615        let mut child = None;
10616        let err = set_child_enabled(
10617            &spec,
10618            &runtime,
10619            &registry,
10620            &supervisor.process_liveness,
10621            &snapshot,
10622            &mut child,
10623            true,
10624        )
10625        .await
10626        .unwrap_err();
10627        assert!(matches!(
10628            err,
10629            SuperviseError::RegistrationStillActive { .. }
10630        ));
10631        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10632        assert!(child.is_none());
10633        registry.deregister_connection(connection).unwrap();
10634        let err = set_child_enabled(
10635            &spec,
10636            &runtime,
10637            &registry,
10638            &supervisor.process_liveness,
10639            &snapshot,
10640            &mut child,
10641            true,
10642        )
10643        .await
10644        .unwrap_err();
10645        assert!(
10646            matches!(err, SuperviseError::Spawn { .. }),
10647            "second enable must attempt a spawn: {err}"
10648        );
10649        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10650    }
10651
10652    const INCUMBENT: u64 = 1;
10653    const CANDIDATE: u64 = 2;
10654
10655    fn swapped_registry() -> Arc<Registry> {
10656        let registry = Arc::new(Registry::default());
10657        let manifest = ModuleManifest::builder("m", "0.1.0").build();
10658        registry
10659            .register_with_control_ops(
10660                manifest.clone(),
10661                1,
10662                ConnectionId::new(INCUMBENT),
10663                Vec::new(),
10664            )
10665            .unwrap();
10666        registry
10667            .register_candidate_with_control_ops(
10668                manifest,
10669                1,
10670                ConnectionId::new(CANDIDATE),
10671                Vec::new(),
10672            )
10673            .unwrap();
10674        registry
10675    }
10676
10677    /// After a promotion the id's active slot is held by the new process, so an
10678    /// id-keyed wait for the incumbent's release can never succeed; the
10679    /// connection-keyed wait completes as soon as the incumbent deregisters.
10680    #[tokio::test]
10681    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
10682        let registry = swapped_registry();
10683        registry.promote_candidate("m").unwrap().unwrap();
10684
10685        assert!(matches!(
10686            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
10687            Err(SuperviseError::RegistrationStillActive { .. })
10688        ));
10689
10690        // Still held while the incumbent's connection has not deregistered.
10691        assert!(matches!(
10692            wait_for_slot_registration_release(
10693                &registry,
10694                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
10695                Duration::from_millis(50),
10696            )
10697            .await,
10698            Err(SuperviseError::RegistrationStillActive { .. })
10699        ));
10700
10701        let releaser = Arc::clone(&registry);
10702        let release = tokio::spawn(async move {
10703            sleep(Duration::from_millis(20)).await;
10704            releaser
10705                .deregister_connection(ConnectionId::new(INCUMBENT))
10706                .unwrap();
10707            notify_registration_release();
10708        });
10709        wait_for_slot_registration_release(
10710            &registry,
10711            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
10712            Duration::from_secs(5),
10713        )
10714        .await
10715        .expect("the incumbent's own registration is released");
10716        release.await.unwrap();
10717        assert!(registry.get_module("m").unwrap().is_some());
10718    }
10719
10720    /// The candidate slot is waited on separately from the active slot: the
10721    /// incumbent's registration neither holds up nor stands in for it.
10722    #[tokio::test]
10723    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
10724        let registry = swapped_registry();
10725        assert!(matches!(
10726            wait_for_slot_registration_release(
10727                &registry,
10728                RegistrationSlot::Candidate("m"),
10729                Duration::from_millis(50),
10730            )
10731            .await,
10732            Err(SuperviseError::RegistrationStillActive { .. })
10733        ));
10734        registry
10735            .deregister_connection(ConnectionId::new(CANDIDATE))
10736            .unwrap();
10737        wait_for_slot_registration_release(
10738            &registry,
10739            RegistrationSlot::Candidate("m"),
10740            Duration::from_millis(50),
10741        )
10742        .await
10743        .expect("a candidate slot with no candidate is released");
10744        assert!(registry
10745            .registration(RegistrationSlot::Active("m"))
10746            .unwrap()
10747            .is_some());
10748    }
10749}
10750
10751fn classify_exit(status: &ExitStatus) -> ExitReport {
10752    ExitReport {
10753        kind: if status.success() {
10754            ExitKind::Clean
10755        } else {
10756            ExitKind::Crash
10757        },
10758        code: status.code(),
10759        signal: exit_signal(status),
10760        at_ms: unix_ms_now(),
10761    }
10762}
10763
10764/// The terminal record for a module whose `wait()` call itself errored (e.g. the
10765/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
10766/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
10767/// disposition still must be `Failed` so the terminal ring is not silently missing
10768/// an entry, matching what `fail_snapshot` records for this same arm.
10769fn wait_error_exit_report() -> ExitReport {
10770    ExitReport {
10771        kind: ExitKind::Crash,
10772        code: None,
10773        signal: None,
10774        at_ms: unix_ms_now(),
10775    }
10776}
10777
10778#[cfg(unix)]
10779fn exit_signal(status: &ExitStatus) -> Option<i32> {
10780    use std::os::unix::process::ExitStatusExt;
10781
10782    status.signal()
10783}
10784
10785#[cfg(not(unix))]
10786fn exit_signal(_status: &ExitStatus) -> Option<i32> {
10787    None
10788}
10789
10790/// Give an operator-touched module its full crash budget back.
10791///
10792/// Named for the counter it used to zero; it now empties the in-window ring,
10793/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
10794/// ledger of what happened survives every operator action.
10795fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
10796    update_snapshot(snapshot, Some(module_id), |state| {
10797        state.clear_crash_restarts();
10798    })
10799}
10800
10801fn set_running(
10802    snapshot: &SharedSnapshot,
10803    child: &SupervisedChild,
10804    module_id: &str,
10805    spawn_events: &SpawnEventFeed,
10806) -> Result<(), SuperviseError> {
10807    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
10808        module_id: Some(module_id.to_string()),
10809    })?;
10810    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
10811    if std::mem::take(&mut state.coalesced_restart_pending) {
10812        let generation = state.spawn_generation;
10813        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
10814    }
10815    state.drain_disposition_detail = None;
10816    state.spawn_failure = None;
10817    state.waiting_for_previous_pid = None;
10818    state.waiting_for_previous_detail = None;
10819    // Every caller of this is a plain spawn, which always uses the primary key;
10820    // a promoted swap candidate sets the flag itself after this returns.
10821    state.in_alternate_slot = false;
10822    state.configuration_updated_since_spawn = false;
10823    state.spawned_protocol = Some(child.protocol);
10824    state.state = ModuleState::Running;
10825    state.enabled = true;
10826    state.process_alive = true;
10827    state.pid = child.id();
10828    #[cfg(windows)]
10829    {
10830        state.pipe_consumed = child
10831            .nonce_delivery
10832            .as_ref()
10833            .map(|delivery| delivery.consumed());
10834    }
10835    #[cfg(target_os = "macos")]
10836    {
10837        state.report_ready = Some(Arc::clone(&child.report_ready));
10838    }
10839    state.spawned_at_ms = Some(child.spawned_at_ms);
10840    state.exec_outcome = Some(Arc::clone(&child.exec.0));
10841    state.spawned_from = Some(child.spawned_from.clone());
10842    state.spawned_file_identity = child.spawned_file_identity.clone();
10843    state.process_start_time = child.process_start_time;
10844    Ok(())
10845}
10846
10847fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
10848    state.waiting_for_previous_pid = None;
10849    state.waiting_for_previous_detail = None;
10850    state.process_alive = false;
10851    state.spawned_protocol = None;
10852    state.pid = None;
10853    #[cfg(windows)]
10854    {
10855        state.pipe_consumed = None;
10856    }
10857    #[cfg(target_os = "macos")]
10858    {
10859        state.report_ready = None;
10860    }
10861    state.spawned_at_ms = None;
10862    state.exec_outcome = None;
10863    state.spawned_from = None;
10864    state.spawned_file_identity = None;
10865    state.process_start_time = None;
10866    state.deliberate_severance = None;
10867}
10868
10869#[cfg(test)]
10870fn record_deliberate_severance(
10871    snapshot: &SharedSnapshot,
10872    identity: ProcessIdentity,
10873) -> Result<(), SuperviseError> {
10874    update_snapshot(snapshot, None, |state| {
10875        state.deliberate_severance = Some(identity);
10876    })
10877}
10878
10879fn apply_deliberate_severance_marker(
10880    snapshot: &SharedSnapshot,
10881    exited_identity: Option<ProcessIdentity>,
10882    mut exit_report: ExitReport,
10883) -> ExitReport {
10884    let marker = lock_snapshot(snapshot)
10885        .ok()
10886        .and_then(|mut state| state.deliberate_severance.take());
10887    if marker.is_some() && marker == exited_identity {
10888        exit_report.kind = ExitKind::DeliberateSeverance;
10889    }
10890    exit_report
10891}
10892
10893fn classify_reaped_child_exit(
10894    snapshot: &SharedSnapshot,
10895    child: &SupervisedChild,
10896    status: &ExitStatus,
10897) -> ExitReport {
10898    let _ = update_snapshot(snapshot, None, |state| {
10899        state.reaped_pid = Some(child.pid);
10900        state.spawn_failure = child.spawn_failure.clone();
10901    });
10902    apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
10903}
10904
10905fn fail_snapshot(
10906    snapshot: &SharedSnapshot,
10907    module_id: Option<&str>,
10908    last_exit: Option<ExitReport>,
10909) {
10910    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
10911        state.state = ModuleState::Failed;
10912        clear_current_process_facts(state);
10913        if let Some(last_exit) = last_exit {
10914            state.last_exit = Some(last_exit);
10915        }
10916    }) {
10917        error!(error = %err, "failed to mark supervisor state failed");
10918    }
10919}
10920
10921fn update_snapshot(
10922    snapshot: &SharedSnapshot,
10923    module_id: Option<&str>,
10924    update: impl FnOnce(&mut SupervisorSnapshot),
10925) -> Result<(), SuperviseError> {
10926    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
10927        module_id: module_id.map(ToOwned::to_owned),
10928    })?;
10929    update(&mut state);
10930    Ok(())
10931}
10932
10933const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
10934
10935fn lock_snapshot_for_control<'a>(
10936    snapshot: &'a SharedSnapshot,
10937    module_id: &str,
10938    caller: &'static str,
10939) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
10940    let started_at = Instant::now();
10941    let guard = lock_snapshot(snapshot)?;
10942    let waited = started_at.elapsed();
10943    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
10944        warn!(
10945            module_id = %module_id,
10946            waited_ms = waited.as_millis() as u64,
10947            caller = %caller,
10948            "slow snapshot lock"
10949        );
10950    }
10951    Ok(guard)
10952}
10953
10954fn lock_snapshot(
10955    snapshot: &SharedSnapshot,
10956) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
10957    snapshot
10958        .lock()
10959        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
10960}
10961
10962#[cfg(test)]
10963mod terminal_history_tests {
10964    use std::{
10965        path::PathBuf,
10966        sync::Arc,
10967        time::{Duration, Instant},
10968    };
10969
10970    use tokio::{sync::mpsc, time::sleep};
10971
10972    use super::{
10973        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
10974        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
10975        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
10976        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
10977        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
10978        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
10979        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
10980        SupervisorHealthStatus, SupervisorSnapshot,
10981    };
10982    // The supervisor's clock, distinct from the `std::time::Instant` these tests
10983    // use for their own wall-clock deadlines: crash-restart instants must be on
10984    // the same clock the production code stamps them with, which is tokio's (and
10985    // is what `start_paused` tests can move).
10986    use super::Instant as ClockInstant;
10987    use crate::{
10988        registry::Registry,
10989        terminal_ring::{TerminalRing, TerminalRingConfig},
10990    };
10991    use std::sync::Mutex;
10992    use subc_control::TerminalDisposition;
10993
10994    /// See the twin in `control.rs` for why this derives the path from
10995    /// `current_exe()` and why the existence check is here: `--lib` alone does
10996    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
10997    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
10998    pub(super) fn fake_aft_stub_path() -> PathBuf {
10999        let mut path = std::env::current_exe().expect("current_exe available in tests");
11000        path.pop();
11001        path.pop();
11002        path.push(if cfg!(windows) {
11003            "fake-aft-stub.exe"
11004        } else {
11005            "fake-aft-stub"
11006        });
11007        assert!(
11008            path.exists(),
11009            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11010             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11011            path.display()
11012        );
11013        path
11014    }
11015
11016    #[test]
11017    fn reserved_never_spawned_refuses_every_hello() {
11018        // The canary hole: a reserved id whose module has never spawned had NO
11019        // gate entry and admitted anyone -- the reservation protected the nonce
11020        // holder, not the NAME. Now the entry is present with no legitimate
11021        // holder and refuses all comers.
11022        let supervisor = SupervisorHandle::default();
11023        supervisor.apply_identity_configuration(&ModuleSpec {
11024            module_id: "never-spawned".to_string(),
11025            program: PathBuf::from("/usr/bin/false"),
11026            args: Vec::new(),
11027            env: Vec::new(),
11028            reserved: true,
11029            reserved_prefixes: Vec::new(),
11030            protocol: ModuleProtocol::Subc,
11031            overlap: Default::default(),
11032        });
11033        assert!(
11034            supervisor
11035                .reserved_hello_rejection("never-spawned", Some("any-forged-nonce"))
11036                .is_some(),
11037            "forged nonce must refuse on a reserved never-spawned id"
11038        );
11039        assert!(
11040            supervisor
11041                .reserved_hello_rejection("never-spawned", None)
11042                .is_some(),
11043            "absent nonce must refuse on a reserved never-spawned id"
11044        );
11045        // And a real spawn nonce minted later admits exactly that nonce.
11046        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11047        supervisor.apply_identity_configuration(&ModuleSpec {
11048            module_id: "never-spawned".to_string(),
11049            program: PathBuf::from("/usr/bin/false"),
11050            args: Vec::new(),
11051            env: Vec::new(),
11052            reserved: true,
11053            reserved_prefixes: Vec::new(),
11054            protocol: ModuleProtocol::Subc,
11055            overlap: Default::default(),
11056        });
11057        assert!(supervisor
11058            .reserved_hello_rejection("never-spawned", Some("minted"))
11059            .is_none());
11060        assert!(supervisor
11061            .reserved_hello_rejection("never-spawned", Some("forged"))
11062            .is_some());
11063    }
11064
11065    /// Put `count` crash restarts on a snapshot's ring as if they had all just
11066    /// happened, which is what "spent budget" looks like to every reader.
11067    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11068        let now = ClockInstant::now();
11069        for _ in 0..count {
11070            state.crash_restarts.push_back(now);
11071        }
11072    }
11073
11074    /// Age the oldest recorded restart out of `window`, standing in for the hours
11075    /// that would otherwise have to pass. Injecting the instant is the point: a
11076    /// test that slept a real window would take ten minutes and still prove less.
11077    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11078        let aged = state
11079            .crash_restarts
11080            .front()
11081            .expect("a crash restart must be recorded before it can be aged")
11082            .checked_sub(window + Duration::from_secs(1))
11083            .expect("the test clock is far enough from its origin to age an instant");
11084        state.crash_restarts[0] = aged;
11085    }
11086
11087    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11088        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11089        seed_crash_restarts(&mut state, count);
11090        state
11091    }
11092
11093    #[test]
11094    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11095        let policy = RestartPolicy::new(3, Duration::ZERO);
11096        let now = ClockInstant::now();
11097        assert!(daemon_will_restart(
11098            &mut snapshot_with_restarts(true, 2),
11099            &policy,
11100            now
11101        ));
11102        assert!(!daemon_will_restart(
11103            &mut snapshot_with_restarts(true, 3),
11104            &policy,
11105            now
11106        ));
11107        assert!(!daemon_will_restart(
11108            &mut snapshot_with_restarts(false, 0),
11109            &policy,
11110            now
11111        ));
11112    }
11113
11114    #[test]
11115    fn crash_restart_backoff_escalates_with_in_window_count() {
11116        let policy = RestartPolicy::new(4, Duration::from_millis(100))
11117            .with_max_backoff(Duration::from_secs(30));
11118        let now = ClockInstant::now();
11119        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11120        let schedules = (0..4)
11121            .map(|_| {
11122                state
11123                    .next_crash_restart(&policy, now)
11124                    .expect("the test policy allows four crash restarts")
11125            })
11126            .collect::<Vec<_>>();
11127
11128        assert_eq!(
11129            schedules
11130                .iter()
11131                .map(|schedule| schedule.restart_in_window)
11132                .collect::<Vec<_>>(),
11133            vec![0, 1, 2, 3]
11134        );
11135        assert_eq!(
11136            schedules
11137                .iter()
11138                .map(|schedule| schedule.delay)
11139                .collect::<Vec<_>>(),
11140            vec![
11141                Duration::from_millis(100),
11142                Duration::from_secs(1),
11143                Duration::from_secs(10),
11144                Duration::from_secs(30),
11145            ]
11146        );
11147    }
11148
11149    #[test]
11150    fn crash_restart_backoff_resets_after_ring_clear() {
11151        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11152        let now = ClockInstant::now();
11153        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11154        assert_eq!(
11155            state.next_crash_restart(&policy, now).unwrap().delay,
11156            Duration::from_millis(100)
11157        );
11158        assert_eq!(
11159            state.next_crash_restart(&policy, now).unwrap().delay,
11160            Duration::from_secs(1)
11161        );
11162
11163        state.clear_crash_restarts();
11164        let schedule = state
11165            .next_crash_restart(&policy, now)
11166            .expect("a cleared ring must allow another restart");
11167        assert_eq!(schedule.restart_in_window, 0);
11168        assert_eq!(schedule.delay, Duration::from_millis(100));
11169    }
11170
11171    #[test]
11172    fn crash_restart_backoff_ignores_aged_restarts() {
11173        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11174        let now = ClockInstant::now();
11175        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11176        state
11177            .next_crash_restart(&policy, now)
11178            .expect("the first restart is allowed");
11179        state
11180            .next_crash_restart(&policy, now)
11181            .expect("the second restart is allowed");
11182        state.crash_restarts[0] = now
11183            .checked_sub(policy.window + Duration::from_secs(1))
11184            .expect("the fake clock can age a restart past the window");
11185
11186        let schedule = state
11187            .next_crash_restart(&policy, now)
11188            .expect("an aged restart must release its slot");
11189        assert_eq!(schedule.restart_in_window, 1);
11190        assert_eq!(schedule.delay, Duration::from_secs(1));
11191        assert_eq!(state.crash_restarts.len(), 2);
11192    }
11193
11194    /// The budget is a rate: the same three spent restarts refuse a respawn
11195    /// while they are recent and allow one once they have aged past the window.
11196    /// Nothing about the module changed in between, which is the whole point.
11197    #[test]
11198    fn a_budget_spent_before_the_window_no_longer_refuses() {
11199        let policy = RestartPolicy::new(3, Duration::ZERO);
11200        let mut state = snapshot_with_restarts(true, 3);
11201        let now = ClockInstant::now();
11202        assert!(!daemon_will_restart(&mut state, &policy, now));
11203
11204        assert!(daemon_will_restart(
11205            &mut state,
11206            &policy,
11207            now + policy.window + Duration::from_secs(1)
11208        ));
11209        assert!(
11210            state.crash_restarts.is_empty(),
11211            "reading the budget must drop the instants that left the window"
11212        );
11213    }
11214
11215    fn module_with_recovery_snapshot(
11216        state: ModuleState,
11217        enabled: bool,
11218        restart_count: u32,
11219    ) -> SupervisedModule {
11220        let registry = Arc::new(Registry::default());
11221        let supervisor =
11222            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
11223        let runtime = supervisor.runtime_config();
11224        let spec = ModuleSpec {
11225            module_id: "recovery-snapshot".to_string(),
11226            program: fake_aft_stub_path(),
11227            args: Vec::new(),
11228            env: Vec::new(),
11229            reserved: false,
11230            reserved_prefixes: Vec::new(),
11231            protocol: ModuleProtocol::Subc,
11232            overlap: Default::default(),
11233        };
11234        let mut snapshot = SupervisorSnapshot::new(state, enabled);
11235        seed_crash_restarts(&mut snapshot, restart_count);
11236        // These tests read synthetic snapshots. A real child and monitor would
11237        // race those reads by replacing the requested state during startup.
11238        let (commands, _rx) = mpsc::channel(4);
11239        SupervisedModule {
11240            inner: Arc::new(SupervisedModuleInner {
11241                module_id: spec.module_id.clone(),
11242                registry,
11243                snapshot: Arc::new(Mutex::new(snapshot)),
11244                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11245                    spec,
11246                    health: runtime.health,
11247                })),
11248                stderr_ring: runtime.stderr_ring,
11249                terminal_ring: runtime.terminal_ring,
11250                commands,
11251                monitor: Mutex::new(None),
11252                restart_policy: runtime.restart_policy,
11253                effective_drain_timeout: runtime.effective_drain_timeout,
11254                provenance_probe: supervisor.provenance_probe.clone(),
11255            }),
11256        }
11257    }
11258
11259    #[cfg(target_os = "linux")]
11260    #[tokio::test]
11261    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11262        let supervisor =
11263            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11264                .with_cgroup_placement(None);
11265        let result = supervisor.spawn(ModuleSpec {
11266            module_id: "no-cgroup-placement".to_string(),
11267            program: fake_aft_stub_path(),
11268            args: Vec::new(),
11269            env: Vec::new(),
11270            reserved: false,
11271            reserved_prefixes: Vec::new(),
11272            protocol: ModuleProtocol::Subc,
11273            overlap: Default::default(),
11274        });
11275
11276        assert!(
11277            result.is_ok(),
11278            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11279        );
11280    }
11281
11282    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11283    async fn undecided_snapshot_uses_shared_restart_predicate() {
11284        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11285            .will_recover_after_connection_loss()
11286            .unwrap());
11287        assert!(
11288            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11289                .will_recover_after_connection_loss()
11290                .unwrap()
11291        );
11292    }
11293
11294    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11295    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11296        assert!(
11297            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11298                .will_recover_after_connection_loss()
11299                .unwrap()
11300        );
11301    }
11302
11303    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11304    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11305        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11306            .will_recover_after_connection_loss()
11307            .unwrap());
11308        assert!(
11309            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11310                .will_recover_after_connection_loss()
11311                .unwrap()
11312        );
11313    }
11314
11315    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11316    async fn warming_snapshot_is_limited_to_startup_phases() {
11317        for state in [
11318            ModuleState::Starting,
11319            ModuleState::Running,
11320            ModuleState::Restarting,
11321        ] {
11322            assert!(
11323                module_with_recovery_snapshot(state, true, 0)
11324                    .is_warming()
11325                    .unwrap(),
11326                "{state:?} should be warming"
11327            );
11328        }
11329        for state in [
11330            ModuleState::Unresponsive,
11331            ModuleState::Draining,
11332            ModuleState::Stopped,
11333            ModuleState::Failed,
11334            ModuleState::Disabled,
11335        ] {
11336            assert!(
11337                !module_with_recovery_snapshot(state, true, 0)
11338                    .is_warming()
11339                    .unwrap(),
11340                "{state:?} should not be warming"
11341            );
11342        }
11343    }
11344
11345    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11346    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11347        let registry = Arc::new(Registry::default());
11348        let supervisor =
11349            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
11350        let module = supervisor
11351            .spawn(ModuleSpec {
11352                module_id: "terminal-history".to_string(),
11353                program: fake_aft_stub_path(),
11354                args: Vec::new(),
11355                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11356                reserved: false,
11357                reserved_prefixes: Vec::new(),
11358                protocol: ModuleProtocol::Subc,
11359                overlap: Default::default(),
11360            })
11361            .unwrap();
11362
11363        let deadline = Instant::now() + Duration::from_secs(5);
11364        loop {
11365            let history = module.terminal_history();
11366            if history.entries.len() == 2 {
11367                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11368                assert_eq!(history.dropped, 0);
11369                assert_eq!(
11370                    history
11371                        .entries
11372                        .iter()
11373                        .map(|entry| entry.exit_code)
11374                        .collect::<Vec<_>>(),
11375                    vec![Some(23), Some(23)]
11376                );
11377                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11378                return;
11379            }
11380            assert!(
11381                Instant::now() < deadline,
11382                "module did not retain two terminal exits: {history:?}"
11383            );
11384            sleep(Duration::from_millis(10)).await;
11385        }
11386    }
11387
11388    /// A disable issued while a crash respawn is still backing off must preempt
11389    /// that respawn: the operator's stop wins, the disable must not queue behind
11390    /// the backoff, and the module must never come back up afterwards.
11391    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11392    async fn disable_during_crash_backoff_cancels_pending_respawn() {
11393        let backoff = Duration::from_secs(2);
11394        let supervisor = Supervisor::new_for_test(
11395            Arc::new(Registry::default()),
11396            RestartPolicy::new(10, backoff),
11397        );
11398        let module = supervisor
11399            .spawn(ModuleSpec {
11400                module_id: "disable-during-backoff".to_string(),
11401                program: fake_aft_stub_path(),
11402                args: Vec::new(),
11403                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11404                reserved: false,
11405                reserved_prefixes: Vec::new(),
11406                protocol: ModuleProtocol::Subc,
11407                overlap: Default::default(),
11408            })
11409            .unwrap();
11410
11411        // Wait for the first crash to put the module into its backoff window.
11412        let deadline = Instant::now() + Duration::from_secs(5);
11413        loop {
11414            if module.status().unwrap().state == ModuleState::Restarting {
11415                break;
11416            }
11417            assert!(
11418                Instant::now() < deadline,
11419                "module never entered the crash backoff"
11420            );
11421            sleep(Duration::from_millis(10)).await;
11422        }
11423
11424        let started = Instant::now();
11425        module.set_enabled(false).await.unwrap();
11426        let waited = started.elapsed();
11427
11428        assert!(
11429            waited < backoff / 2,
11430            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
11431        );
11432        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
11433
11434        // Outlast the backoff: the respawn it was counting down to must never run.
11435        sleep(backoff + Duration::from_millis(500)).await;
11436        let status = module.status().unwrap();
11437        assert_eq!(status.state, ModuleState::Disabled);
11438        assert_eq!(
11439            status.spawn_generation, 1,
11440            "module respawned after the operator disabled it"
11441        );
11442    }
11443
11444    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
11445    /// the shape of nats-server, the program this rule exists for.
11446    #[cfg(unix)]
11447    fn protocol_none_sigterm_exits_clean_spec(
11448        module_id: &str,
11449        dir: &std::path::Path,
11450    ) -> (ModuleSpec, PathBuf, PathBuf) {
11451        let ready = dir.join("ready");
11452        let marker = dir.join("sigterm");
11453        let spec = ModuleSpec {
11454            module_id: module_id.to_string(),
11455            program: fake_aft_stub_path(),
11456            args: Vec::new(),
11457            env: vec![
11458                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
11459                (
11460                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
11461                    marker.display().to_string(),
11462                ),
11463                (
11464                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
11465                    ready.display().to_string(),
11466                ),
11467            ],
11468            reserved: false,
11469            reserved_prefixes: Vec::new(),
11470            protocol: ModuleProtocol::None,
11471            overlap: Default::default(),
11472        };
11473        (spec, ready, marker)
11474    }
11475
11476    /// Wait for a file the child writes, so a signal is never sent before the
11477    /// child's SIGTERM handler is installed (the default disposition would
11478    /// kill it by signal and the exit would not be clean).
11479    #[cfg(unix)]
11480    async fn wait_for_file(path: &std::path::Path) {
11481        let deadline = Instant::now() + Duration::from_secs(10);
11482        while !path.exists() {
11483            assert!(
11484                Instant::now() < deadline,
11485                "{} never appeared",
11486                path.display()
11487            );
11488            sleep(Duration::from_millis(10)).await;
11489        }
11490    }
11491
11492    /// A protocol-none module that exits 0 because something OUTSIDE the
11493    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
11494    /// the crash-path disposition rather than `stopped`.
11495    #[cfg(unix)]
11496    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11497    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
11498        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
11499        let (spec, ready, marker) =
11500            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
11501        let supervisor = Supervisor::new_for_test(
11502            Arc::new(Registry::default()),
11503            RestartPolicy::new(3, Duration::ZERO),
11504        );
11505        let module = supervisor.spawn(spec).unwrap();
11506        wait_for_file(&ready).await;
11507        // The ready file proves the child installed its SIGTERM handler, not
11508        // that the supervisor has processed the privacy trampoline's exec
11509        // acknowledgement. On macOS status withholds the pid until then.
11510        let deadline = Instant::now() + Duration::from_secs(10);
11511        let first_pid = loop {
11512            let status = module.status().unwrap();
11513            if status.state == ModuleState::Running {
11514                if let Some(pid) = status.pid {
11515                    break pid;
11516                }
11517            }
11518            assert!(
11519                Instant::now() < deadline,
11520                "a running module must report its pid after exec confirmation: {status:?}"
11521            );
11522            sleep(Duration::from_millis(10)).await;
11523        };
11524
11525        rustix::process::kill_process(
11526            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
11527            rustix::process::Signal::TERM,
11528        )
11529        .unwrap();
11530
11531        let deadline = Instant::now() + Duration::from_secs(10);
11532        let respawned = loop {
11533            let status = module.status().unwrap();
11534            if status.state == ModuleState::Running
11535                && status.pid.is_some_and(|pid| pid != first_pid)
11536            {
11537                break status;
11538            }
11539            assert!(
11540                Instant::now() < deadline,
11541                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
11542            );
11543            sleep(Duration::from_millis(10)).await;
11544        };
11545        assert_eq!(respawned.spawn_generation, 2);
11546        assert!(
11547            marker.exists(),
11548            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
11549        );
11550
11551        let history = module.terminal_history();
11552        assert_eq!(history.entries.len(), 1, "{history:?}");
11553        let entry = &history.entries[0];
11554        assert_eq!(entry.exit_code, Some(0));
11555        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
11556        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
11557
11558        module.stop().await.unwrap();
11559    }
11560
11561    /// Repeated unrequested clean exits of a protocol-none module spend the
11562    /// restart budget exactly as crashes do, and the module ends `failed` with
11563    /// the budget named.
11564    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11565    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
11566        let supervisor = Supervisor::new_for_test(
11567            Arc::new(Registry::default()),
11568            RestartPolicy::new(1, Duration::ZERO),
11569        );
11570        let module = supervisor
11571            .spawn(ModuleSpec {
11572                module_id: "none-clean-exit-budget".to_string(),
11573                program: fake_aft_stub_path(),
11574                args: Vec::new(),
11575                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
11576                reserved: false,
11577                reserved_prefixes: Vec::new(),
11578                protocol: ModuleProtocol::None,
11579                overlap: Default::default(),
11580            })
11581            .unwrap();
11582
11583        // Failed follows the terminal write; this deadline only bounds a hang,
11584        // not an assumed duration for the two launches or their exit recording.
11585        let deadline = Instant::now() + Duration::from_secs(10);
11586        loop {
11587            let status = module.status().unwrap();
11588            if status.state == ModuleState::Failed {
11589                break;
11590            }
11591            assert!(
11592                Instant::now() < deadline,
11593                "module never exhausted its budget: {status:?} {:?}",
11594                module.terminal_history()
11595            );
11596            sleep(Duration::from_millis(10)).await;
11597        }
11598        let history = module.terminal_history();
11599        assert_eq!(
11600            history
11601                .entries
11602                .iter()
11603                .map(|entry| (entry.exit_code, entry.disposition.clone()))
11604                .collect::<Vec<_>>(),
11605            vec![
11606                (Some(0), TerminalDisposition::Restarting),
11607                (Some(0), TerminalDisposition::Failed),
11608            ]
11609        );
11610        let detail = history.entries[1]
11611            .disposition_detail
11612            .as_deref()
11613            .expect("a budget failure names the budget");
11614        assert!(detail.contains("max_restarts=1"), "{detail}");
11615        assert_eq!(module.status().unwrap().spawn_generation, 2);
11616    }
11617
11618    #[test]
11619    fn restart_budget_failure_is_published_after_its_terminal_record() {
11620        use std::io::{Read, Write};
11621        let supervisor = Supervisor::new_for_test(
11622            Arc::new(Registry::default()),
11623            RestartPolicy::new(0, Duration::ZERO),
11624        );
11625        let runtime = supervisor.runtime_config();
11626        let spec = ModuleSpec {
11627            protocol: ModuleProtocol::None,
11628            ..windowed_crash_spec("budget-publication-order")
11629        };
11630        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
11631            ModuleState::Running,
11632            true,
11633        )));
11634        lock_snapshot(&snapshot).unwrap().process_alive = true;
11635        let reaped_snapshot = snapshot.clone();
11636        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11637        let history = runtime.terminal_ring.clone();
11638        let reap = std::thread::spawn(move || {
11639            tokio::runtime::Builder::new_current_thread()
11640                .enable_all()
11641                .build()
11642                .unwrap()
11643                .block_on(on_child_exit(
11644                    &spec,
11645                    runtime.restart_policy,
11646                    &supervisor.registry,
11647                    &reaped_snapshot,
11648                    &runtime.terminal_ring,
11649                    &runtime.spawn_events,
11650                    &runtime.child_roster,
11651                    ExitReport {
11652                        kind: ExitKind::Clean,
11653                        code: Some(0),
11654                        signal: None,
11655                        at_ms: 1,
11656                    },
11657                ))
11658        });
11659        let mut reached = [0];
11660        barrier.read_exact(&mut reached).unwrap();
11661        assert_eq!(&reached, b"R");
11662        let (before_state, before_alive) = {
11663            let state = lock_snapshot(&snapshot).unwrap();
11664            (state.state, state.process_alive)
11665        };
11666        let before_history = history.lock().unwrap().snapshot();
11667        barrier.write_all(b"X").unwrap();
11668        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
11669        assert!(before_history.entries.is_empty());
11670        assert_ne!(
11671            before_state,
11672            ModuleState::Failed,
11673            "Failed was visible before its terminal record could be written"
11674        );
11675        assert!(before_alive);
11676        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11677        let history = history.lock().unwrap().snapshot();
11678        assert_eq!(history.entries.len(), 1);
11679        assert_eq!(history.entries[0].exit_code, Some(0));
11680        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11681    }
11682
11683    #[test]
11684    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
11685        use std::io::{Read, Write};
11686
11687        let supervisor = Supervisor::new_for_test(
11688            Arc::new(Registry::default()),
11689            RestartPolicy::new(3, Duration::ZERO),
11690        );
11691        let mut runtime = supervisor.runtime_config();
11692        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
11693        let snapshot = module.inner.snapshot.clone();
11694        lock_snapshot(&snapshot).unwrap().process_alive = true;
11695        runtime.terminal_ring = module.inner.terminal_ring.clone();
11696        let spec = windowed_crash_spec(module.module_id());
11697        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11698        let reaped_snapshot = snapshot.clone();
11699        let reap = std::thread::spawn(move || {
11700            tokio::runtime::Builder::new_current_thread()
11701                .enable_all()
11702                .build()
11703                .unwrap()
11704                .block_on(on_child_exit(
11705                    &spec,
11706                    runtime.restart_policy,
11707                    &supervisor.registry,
11708                    &reaped_snapshot,
11709                    &runtime.terminal_ring,
11710                    &runtime.spawn_events,
11711                    &runtime.child_roster,
11712                    ExitReport {
11713                        kind: ExitKind::Crash,
11714                        code: Some(2),
11715                        signal: None,
11716                        at_ms: 1,
11717                    },
11718                ))
11719        });
11720
11721        // Hold the crash handler between its budget decision and publication.
11722        // Connection cleanup must still see the recovery that this crash chose,
11723        // even when that recovery consumes the last available restart slot.
11724        let mut reached = [0];
11725        barrier.read_exact(&mut reached).unwrap();
11726        assert_eq!(&reached, b"R");
11727        let before_status = module.status().unwrap();
11728        let before_history = module.terminal_history();
11729        let before_recovers = module.will_recover_after_connection_loss().unwrap();
11730        barrier.write_all(b"X").unwrap();
11731        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
11732
11733        assert_eq!(before_status.state, ModuleState::Running);
11734        assert!(before_status.process_alive);
11735        assert_eq!(before_status.restart_count, 3);
11736        assert!(before_history.entries.is_empty());
11737        assert!(
11738            before_recovers,
11739            "connection cleanup would report a terminal close for a scheduled crash restart"
11740        );
11741        let restarting = module.status().unwrap();
11742        assert_eq!(restarting.state, ModuleState::Restarting);
11743        assert_eq!(restarting.restart_count, 3);
11744        assert!(module.will_recover_after_connection_loss().unwrap());
11745        let history = module.terminal_history();
11746        assert_eq!(history.entries.len(), 1);
11747        assert_eq!(
11748            history.entries[0].disposition,
11749            TerminalDisposition::Restarting
11750        );
11751
11752        // Once the replacement is running, its next crash must be judged
11753        // against the exhausted budget rather than the previous recovery.
11754        update_snapshot(&snapshot, Some(module.module_id()), |state| {
11755            state.state = ModuleState::Running;
11756            state.respawn_pending = false;
11757        })
11758        .unwrap();
11759        assert!(!module.will_recover_after_connection_loss().unwrap());
11760        update_snapshot(&snapshot, Some(module.module_id()), |state| {
11761            state.enabled = false;
11762            state.respawn_pending = true;
11763        })
11764        .unwrap();
11765        assert!(!module.will_recover_after_connection_loss().unwrap());
11766    }
11767
11768    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
11769        use std::net::{TcpListener, TcpStream};
11770        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
11771        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
11772        let (barrier, _) = listener.accept().unwrap();
11773        for stream in [&writer, &barrier] {
11774            stream
11775                .set_read_timeout(Some(Duration::from_secs(10)))
11776                .unwrap();
11777            stream
11778                .set_write_timeout(Some(Duration::from_secs(10)))
11779                .unwrap();
11780        }
11781        *events.1.lock().unwrap() = Some(writer);
11782        barrier
11783    }
11784
11785    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
11786        use std::io::{Read, Write};
11787        let barrier = events.1.lock().unwrap().take();
11788        if let Some(mut barrier) = barrier {
11789            barrier.write_all(b"R").unwrap();
11790            let mut release = [0];
11791            barrier.read_exact(&mut release).unwrap();
11792            assert_eq!(&release, b"X");
11793        }
11794    }
11795
11796    #[test]
11797    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
11798        use std::io::{Read, Write};
11799
11800        let supervisor = Supervisor::new_for_test(
11801            Arc::new(Registry::default()),
11802            RestartPolicy::new(0, Duration::ZERO),
11803        );
11804        let mut runtime = supervisor.runtime_config();
11805        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
11806        let snapshot = module.inner.snapshot.clone();
11807        runtime.terminal_ring = module.inner.terminal_ring.clone();
11808        let spec = ModuleSpec {
11809            module_id: module.module_id().into(),
11810            program: fake_aft_stub_path(),
11811            protocol: ModuleProtocol::None,
11812            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
11813            ..windowed_crash_spec(module.module_id())
11814        };
11815        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11816        let reap = std::thread::spawn(move || {
11817            tokio::runtime::Builder::new_current_thread()
11818                .enable_all()
11819                .build()
11820                .unwrap()
11821                .block_on(async {
11822                    let mut child =
11823                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
11824                    health_restart_child(
11825                        &spec,
11826                        &runtime,
11827                        &supervisor.registry,
11828                        &supervisor.process_liveness,
11829                        &snapshot,
11830                        &mut child,
11831                        SupervisorHealthStatus::Failing,
11832                        Some("health probe did not answer"),
11833                        1,
11834                    )
11835                    .await
11836                    .unwrap();
11837                    assert!(child.is_none());
11838                });
11839        });
11840        // The loopback barrier acknowledges entry into terminal recording by
11841        // health_restart_child's drain, then waits for the test's release byte.
11842        // Status and history are observed while that writer is held, not after
11843        // an elapsed-time guess about when the health action has completed.
11844        let mut reached = [0];
11845        barrier.read_exact(&mut reached).unwrap();
11846        assert_eq!(&reached, b"R");
11847        let before_status = module.status().unwrap();
11848        let before_history = module.terminal_history();
11849        barrier.write_all(b"X").unwrap();
11850        reap.join().unwrap();
11851        assert!(before_history.entries.is_empty());
11852        assert_ne!(
11853            before_status.state,
11854            ModuleState::Failed,
11855            "Failed was visible before its terminal record could be written"
11856        );
11857        assert!(
11858            before_status.process_alive,
11859            "process death was visible before its terminal record could be written"
11860        );
11861        let failed = module.status().unwrap();
11862        assert_eq!(failed.state, ModuleState::Failed);
11863        assert!(!failed.process_alive);
11864        let history = module.terminal_history();
11865        assert_eq!(history.entries.len(), 1);
11866        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11867        assert_eq!(
11868            history.entries[0].disposition_detail.as_deref(),
11869            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
11870        );
11871    }
11872
11873    #[test]
11874    fn observed_exits_publish_process_death_after_terminal_history() {
11875        use std::io::{Read, Write};
11876        for (kind, enabled, shutdown, wait_error, expected_state) in [
11877            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
11878            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
11879            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
11880            (
11881                ExitKind::DeliberateSeverance,
11882                true,
11883                false,
11884                false,
11885                ModuleState::Restarting,
11886            ),
11887            #[cfg(unix)]
11888            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
11889            (ExitKind::Crash, true, false, true, ModuleState::Failed),
11890        ] {
11891            let supervisor = Supervisor::new_for_test(
11892                Arc::new(Registry::default()),
11893                RestartPolicy::new(1, Duration::ZERO),
11894            );
11895            let runtime = supervisor.runtime_config();
11896            #[cfg(unix)]
11897            if shutdown {
11898                runtime.child_roster.close();
11899            }
11900            #[cfg(not(unix))]
11901            let _ = shutdown;
11902            let spec = windowed_crash_spec("exit-publication-order");
11903            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
11904                ModuleState::Running,
11905                enabled,
11906            )));
11907            lock_snapshot(&snapshot).unwrap().process_alive = true;
11908            let reaped_snapshot = snapshot.clone();
11909            let history = runtime.terminal_ring.clone();
11910            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11911            let reap = std::thread::spawn(move || {
11912                if wait_error {
11913                    super::fail_after_wait_error(
11914                        &spec.module_id,
11915                        &reaped_snapshot,
11916                        &runtime.terminal_ring,
11917                        &runtime.spawn_events,
11918                        "wait failed".into(),
11919                    );
11920                } else {
11921                    tokio::runtime::Builder::new_current_thread()
11922                        .enable_all()
11923                        .build()
11924                        .unwrap()
11925                        .block_on(on_child_exit(
11926                            &spec,
11927                            runtime.restart_policy,
11928                            &supervisor.registry,
11929                            &reaped_snapshot,
11930                            &runtime.terminal_ring,
11931                            &runtime.spawn_events,
11932                            &runtime.child_roster,
11933                            ExitReport {
11934                                kind,
11935                                code: Some(0),
11936                                signal: None,
11937                                at_ms: 1,
11938                            },
11939                        ));
11940                }
11941            });
11942            let mut reached = [0];
11943            barrier.read_exact(&mut reached).unwrap();
11944            assert_eq!(&reached, b"R");
11945            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
11946            let before_history = history.lock().unwrap().snapshot();
11947            barrier.write_all(b"X").unwrap();
11948            reap.join().unwrap();
11949            assert!(
11950                before_alive,
11951                "{kind:?}: death was published before recording"
11952            );
11953            assert!(before_history.entries.is_empty());
11954            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
11955            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
11956            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
11957        }
11958    }
11959
11960    #[tokio::test]
11961    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
11962        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
11963        let path = dir.join("terminals.jsonl");
11964        std::fs::create_dir(&path).unwrap();
11965        let supervisor = Supervisor::new_for_test(
11966            Arc::new(Registry::default()),
11967            RestartPolicy::new(0, Duration::ZERO),
11968        )
11969        .with_terminal_journal(path, "health-budget-journal-failure".into());
11970        let runtime = supervisor.runtime_config();
11971        let spec = ModuleSpec {
11972            program: fake_aft_stub_path(),
11973            protocol: ModuleProtocol::None,
11974            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
11975            ..windowed_crash_spec("health-budget-journal-failure")
11976        };
11977        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
11978        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
11979        health_restart_child(
11980            &spec,
11981            &runtime,
11982            &supervisor.registry,
11983            &supervisor.process_liveness,
11984            &snapshot,
11985            &mut child,
11986            SupervisorHealthStatus::Failing,
11987            None,
11988            1,
11989        )
11990        .await
11991        .unwrap();
11992        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11993        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
11994        let history = runtime
11995            .terminal_ring
11996            .lock()
11997            .unwrap()
11998            .durable_history(&spec.module_id);
11999        assert!(history.journal_write_failures > 0);
12000        assert_eq!(history.entries.len(), 1);
12001        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12002    }
12003
12004    #[test]
12005    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12006        let supervisor = Supervisor::new_for_test(
12007            Arc::new(Registry::default()),
12008            RestartPolicy::new(0, Duration::ZERO),
12009        );
12010        let runtime = supervisor.runtime_config();
12011        let spec = ModuleSpec {
12012            program: fake_aft_stub_path(),
12013            protocol: ModuleProtocol::None,
12014            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12015            ..windowed_crash_spec("health-budget-recording-panic")
12016        };
12017        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12018        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12019            tokio::runtime::Builder::new_current_thread()
12020                .enable_all()
12021                .build()
12022                .unwrap()
12023                .block_on(async {
12024                    let mut child =
12025                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12026                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12027                    health_restart_child(
12028                        &spec,
12029                        &runtime,
12030                        &supervisor.registry,
12031                        &supervisor.process_liveness,
12032                        &snapshot,
12033                        &mut child,
12034                        SupervisorHealthStatus::Failing,
12035                        None,
12036                        1,
12037                    )
12038                    .await
12039                    .unwrap();
12040                });
12041        }));
12042        let panic = result.expect_err("recording must still unwind");
12043        assert_eq!(
12044            panic.downcast_ref::<String>().map(String::as_str),
12045            Some("spawn event sequence exhausted")
12046        );
12047        let state = lock_snapshot(&snapshot).unwrap();
12048        assert_eq!(state.state, ModuleState::Failed);
12049        assert!(!state.process_alive);
12050        assert!(state.last_exit.is_some());
12051    }
12052
12053    #[tokio::test]
12054    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12055        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12056        let path = dir.join("terminals.jsonl");
12057        std::fs::create_dir(&path).unwrap();
12058        let supervisor = Supervisor::new_for_test(
12059            Arc::new(Registry::default()),
12060            RestartPolicy::new(0, Duration::ZERO),
12061        )
12062        .with_terminal_journal(path, "budget-journal-failure".into());
12063        let runtime = supervisor.runtime_config();
12064        let spec = windowed_crash_spec("budget-journal-failure");
12065        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12066        assert!(matches!(
12067            on_child_exit(
12068                &spec,
12069                runtime.restart_policy,
12070                &supervisor.registry,
12071                &snapshot,
12072                &runtime.terminal_ring,
12073                &runtime.spawn_events,
12074                &runtime.child_roster,
12075                crash_exit_report(1),
12076            )
12077            .await,
12078            NextAction::Stop { .. }
12079        ));
12080        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12081        let history = runtime
12082            .terminal_ring
12083            .lock()
12084            .unwrap()
12085            .durable_history(&spec.module_id);
12086        assert!(history.journal_write_failures > 0);
12087        assert_eq!(history.entries.len(), 1);
12088        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12089    }
12090
12091    #[test]
12092    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12093        let supervisor = Supervisor::new_for_test(
12094            Arc::new(Registry::default()),
12095            RestartPolicy::new(0, Duration::ZERO),
12096        );
12097        let runtime = supervisor.runtime_config();
12098        let spec = windowed_crash_spec("budget-recording-panic");
12099        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12100        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12101        // Exhausting the event sequence makes emit_exited panic before the
12102        // terminal write, exercising publication on the recording unwind.
12103        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12104        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12105            tokio::runtime::Builder::new_current_thread()
12106                .enable_all()
12107                .build()
12108                .unwrap()
12109                .block_on(on_child_exit(
12110                    &spec,
12111                    runtime.restart_policy,
12112                    &supervisor.registry,
12113                    &snapshot,
12114                    &runtime.terminal_ring,
12115                    &runtime.spawn_events,
12116                    &runtime.child_roster,
12117                    crash_exit_report(1),
12118                ))
12119        }));
12120        let panic = result.err().expect("recording must still unwind");
12121        assert_eq!(
12122            panic.downcast_ref::<String>().map(String::as_str),
12123            Some("spawn event sequence exhausted")
12124        );
12125        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12126    }
12127
12128    /// A stop the supervisor itself requests still stops a protocol-none
12129    /// module, even though the child answers the SIGTERM with exit 0.
12130    #[cfg(unix)]
12131    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12132    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12133        for disable in [false, true] {
12134            let label = if disable {
12135                "none-requested-disable"
12136            } else {
12137                "none-requested-stop"
12138            };
12139            let dir = cortexkit_test_support::ScratchDir::new(label);
12140            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12141            let supervisor = Supervisor::new_for_test(
12142                Arc::new(Registry::default()),
12143                RestartPolicy::new(3, Duration::ZERO),
12144            );
12145            let module = supervisor.spawn(spec).unwrap();
12146            wait_for_file(&ready).await;
12147
12148            if disable {
12149                module.set_enabled(false).await.unwrap();
12150            } else {
12151                module.stop().await.unwrap();
12152            }
12153            assert!(
12154                marker.exists(),
12155                "{label}: the child must have left through its SIGTERM handler with exit 0"
12156            );
12157
12158            // Long enough for a zero-backoff respawn to have happened if the
12159            // exit had been treated as a crash.
12160            sleep(Duration::from_millis(500)).await;
12161            let status = module.status().unwrap();
12162            let expected = if disable {
12163                ModuleState::Disabled
12164            } else {
12165                ModuleState::Stopped
12166            };
12167            assert_eq!(status.state, expected, "{label}");
12168            assert_eq!(
12169                status.spawn_generation, 1,
12170                "{label}: respawned after a requested stop"
12171            );
12172            let history = module.terminal_history();
12173            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12174            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12175            assert_ne!(
12176                history.entries[0].disposition,
12177                TerminalDisposition::Restarting,
12178                "{label}"
12179            );
12180        }
12181    }
12182
12183    /// A subc-wire module that exits 0 on its own is still a stop: the
12184    /// protocol-none rule must not reach it.
12185    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12186    async fn subc_wire_clean_exit_is_still_a_stop() {
12187        let supervisor = Supervisor::new_for_test(
12188            Arc::new(Registry::default()),
12189            RestartPolicy::new(3, Duration::ZERO),
12190        );
12191        let module = supervisor
12192            .spawn(ModuleSpec {
12193                module_id: "wire-clean-exit".to_string(),
12194                program: fake_aft_stub_path(),
12195                args: Vec::new(),
12196                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12197                reserved: false,
12198                reserved_prefixes: Vec::new(),
12199                protocol: ModuleProtocol::Subc,
12200                overlap: Default::default(),
12201            })
12202            .unwrap();
12203
12204        let deadline = Instant::now() + Duration::from_secs(10);
12205        while module.terminal_history().entries.is_empty() {
12206            assert!(Instant::now() < deadline, "module never exited");
12207            sleep(Duration::from_millis(10)).await;
12208        }
12209        // Long enough for a zero-backoff respawn to have happened.
12210        sleep(Duration::from_millis(500)).await;
12211        let status = module.status().unwrap();
12212        assert_eq!(status.state, ModuleState::Stopped);
12213        assert_eq!(status.spawn_generation, 1);
12214        let history = module.terminal_history();
12215        assert_eq!(history.entries.len(), 1, "{history:?}");
12216        assert_eq!(history.entries[0].exit_code, Some(0));
12217        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12218    }
12219
12220    #[cfg(unix)]
12221    #[tokio::test]
12222    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12223        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12224        let record = dir.join("live-children.json");
12225        let supervisor = Supervisor::new_for_test(
12226            Arc::new(Registry::default()),
12227            RestartPolicy::new(0, Duration::ZERO),
12228        );
12229        let mut runtime = supervisor.runtime_config();
12230        runtime.child_roster.record_to(record.clone());
12231        let gate = Arc::new(super::ReloadExitRecordGate::default());
12232        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12233        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12234        let spec = ModuleSpec {
12235            module_id: "reload-exit-roster".into(),
12236            program: fake_aft_stub_path(),
12237            args: Vec::new(),
12238            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12239            reserved: false,
12240            reserved_prefixes: Vec::new(),
12241            protocol: ModuleProtocol::Subc,
12242            overlap: Default::default(),
12243        };
12244        let mut child = None;
12245        let reload = super::finish_reload_child(
12246            &spec,
12247            &runtime,
12248            &supervisor.registry,
12249            &supervisor.process_liveness,
12250            &snapshot,
12251            &mut child,
12252        );
12253        tokio::pin!(reload);
12254        tokio::select! {
12255            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12256            _ = gate.reached.notified() => {}
12257        }
12258        assert!(runtime
12259            .terminal_ring
12260            .lock()
12261            .unwrap()
12262            .snapshot()
12263            .entries
12264            .is_empty());
12265        assert_eq!(
12266            crate::live_children::read_record(&record).unwrap().len(),
12267            1,
12268            "shutdown must still wait for the reaped child until its terminal record exists"
12269        );
12270        runtime.child_roster.close();
12271        gate.resume.notify_one();
12272        assert!(reload.await.is_err());
12273        assert!(crate::live_children::read_record(&record)
12274            .unwrap()
12275            .is_empty());
12276        let history = runtime.terminal_ring.lock().unwrap().snapshot();
12277        assert_eq!(history.entries.len(), 1);
12278        assert_eq!(
12279            history.entries[0].disposition,
12280            TerminalDisposition::DaemonShutdown
12281        );
12282    }
12283
12284    /// Each restart-producing arm has its own state transition. Keeping their
12285    /// lifetime count assertions adjacent prevents a later new arm from silently
12286    /// spending budget without recording the historical restart.
12287    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12288    async fn every_restart_increment_path_advances_lifetime_count() {
12289        let supervisor = Supervisor::new_for_test(
12290            Arc::new(Registry::default()),
12291            RestartPolicy::new(1, Duration::ZERO),
12292        );
12293        let runtime = supervisor.runtime_config();
12294        let spec = ModuleSpec {
12295            module_id: "lifetime-increment-path".to_string(),
12296            program: PathBuf::from("/unused/lifetime-increment-path"),
12297            args: Vec::new(),
12298            env: Vec::new(),
12299            reserved: false,
12300            reserved_prefixes: Vec::new(),
12301            protocol: ModuleProtocol::Subc,
12302            overlap: Default::default(),
12303        };
12304
12305        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12306        assert!(matches!(
12307            on_child_exit(
12308                &spec,
12309                runtime.restart_policy,
12310                &supervisor.registry,
12311                &crash_snapshot,
12312                &runtime.terminal_ring,
12313                &runtime.spawn_events,
12314                &runtime.child_roster,
12315                ExitReport {
12316                    kind: ExitKind::Crash,
12317                    code: Some(1),
12318                    signal: None,
12319                    at_ms: 1,
12320                },
12321            )
12322            .await,
12323            NextAction::Restart { schedule: _ }
12324        ));
12325        let (crash_restarts, crash_lifetime) = {
12326            let state = lock_snapshot(&crash_snapshot).unwrap();
12327            (state.crash_restarts.len(), state.lifetime_restarts)
12328        };
12329        assert_eq!(crash_restarts, 1);
12330        assert_eq!(crash_lifetime, 1);
12331
12332        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12333        let mut health_child = None;
12334        assert!(matches!(
12335            health_restart_child(
12336                &spec,
12337                &runtime,
12338                &supervisor.registry,
12339                &supervisor.process_liveness,
12340                &health_snapshot,
12341                &mut health_child,
12342                SupervisorHealthStatus::Failing,
12343                None,
12344                2,
12345            )
12346            .await,
12347            Ok(())
12348        ));
12349        assert!(health_child.is_none());
12350        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12351        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12352        let (health_restarts, health_lifetime) = {
12353            let state = lock_snapshot(&health_snapshot).unwrap();
12354            (state.crash_restarts.len(), state.lifetime_restarts)
12355        };
12356        assert_eq!(health_restarts, 1);
12357        assert_eq!(health_lifetime, 1);
12358
12359        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12360        let mut reload_child = None;
12361        assert!(matches!(
12362            handle_reload_spawn_failure(
12363                &spec,
12364                &runtime,
12365                &supervisor.process_liveness,
12366                &reload_snapshot,
12367                &mut reload_child,
12368                "forced reload spawn failure".to_string(),
12369            )
12370            .await,
12371            Err(SuperviseError::ReloadFailed { .. })
12372        ));
12373        let (reload_restarts, reload_lifetime) = {
12374            let state = lock_snapshot(&reload_snapshot).unwrap();
12375            (state.crash_restarts.len(), state.lifetime_restarts)
12376        };
12377        assert_eq!(reload_restarts, 1);
12378        assert_eq!(reload_lifetime, 1);
12379    }
12380
12381    #[tokio::test]
12382    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12383        let supervisor = Supervisor::new_for_test(
12384            Arc::new(Registry::default()),
12385            RestartPolicy::new(3, Duration::ZERO),
12386        );
12387        let runtime = supervisor.runtime_config();
12388        let spec = ModuleSpec {
12389            module_id: "deliberately-severed".to_string(),
12390            program: PathBuf::from("/unused/deliberately-severed"),
12391            args: Vec::new(),
12392            env: Vec::new(),
12393            reserved: false,
12394            reserved_prefixes: Vec::new(),
12395            protocol: ModuleProtocol::Subc,
12396            overlap: Default::default(),
12397        };
12398        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12399        let process = ProcessIdentity {
12400            pid: 41,
12401            start_time: 101,
12402        };
12403        record_deliberate_severance(&snapshot, process).unwrap();
12404        let exit_report = apply_deliberate_severance_marker(
12405            &snapshot,
12406            Some(process),
12407            ExitReport {
12408                kind: ExitKind::Crash,
12409                code: Some(1),
12410                signal: None,
12411                at_ms: 1,
12412            },
12413        );
12414        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12415
12416        assert!(matches!(
12417            on_child_exit(
12418                &spec,
12419                runtime.restart_policy,
12420                &supervisor.registry,
12421                &snapshot,
12422                &runtime.terminal_ring,
12423                &runtime.spawn_events,
12424                &runtime.child_roster,
12425                exit_report,
12426            )
12427            .await,
12428            NextAction::Restart { schedule: _ }
12429        ));
12430        let state = lock_snapshot(&snapshot).unwrap();
12431        assert_eq!(state.lifetime_restarts, 1);
12432        assert_eq!(state.crash_restarts.len(), 0);
12433    }
12434
12435    #[tokio::test]
12436    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
12437        let supervisor = Supervisor::new_for_test(
12438            Arc::new(Registry::default()),
12439            RestartPolicy::new(3, Duration::ZERO),
12440        );
12441        let runtime = supervisor.runtime_config();
12442        let spec = ModuleSpec {
12443            module_id: "genuine-crash".to_string(),
12444            program: PathBuf::from("/unused/genuine-crash"),
12445            args: Vec::new(),
12446            env: Vec::new(),
12447            reserved: false,
12448            reserved_prefixes: Vec::new(),
12449            protocol: ModuleProtocol::Subc,
12450            overlap: Default::default(),
12451        };
12452        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12453
12454        assert!(matches!(
12455            on_child_exit(
12456                &spec,
12457                runtime.restart_policy,
12458                &supervisor.registry,
12459                &snapshot,
12460                &runtime.terminal_ring,
12461                &runtime.spawn_events,
12462                &runtime.child_roster,
12463                ExitReport {
12464                    kind: ExitKind::Crash,
12465                    code: Some(1),
12466                    signal: None,
12467                    at_ms: 1,
12468                },
12469            )
12470            .await,
12471            NextAction::Restart { schedule: _ }
12472        ));
12473        let state = lock_snapshot(&snapshot).unwrap();
12474        assert_eq!(state.lifetime_restarts, 1);
12475        assert_eq!(state.crash_restarts.len(), 1);
12476    }
12477
12478    fn crash_exit_report(at_ms: u64) -> ExitReport {
12479        ExitReport {
12480            kind: ExitKind::Crash,
12481            code: Some(1),
12482            signal: None,
12483            at_ms,
12484        }
12485    }
12486
12487    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
12488        ModuleSpec {
12489            module_id: module_id.to_string(),
12490            program: PathBuf::from("/unused").join(module_id),
12491            args: Vec::new(),
12492            env: Vec::new(),
12493            reserved: false,
12494            reserved_prefixes: Vec::new(),
12495            protocol: ModuleProtocol::Subc,
12496            overlap: Default::default(),
12497        }
12498    }
12499
12500    /// A real crash loop still stops. Three crashes with nothing aging out spend
12501    /// a budget of two and the third respawn is refused, and both surfaces an
12502    /// operator has -- the log line and the retained terminal record -- name the
12503    /// window rather than only the cap, because `max_restarts=2` alone is what
12504    /// this budget used to mean.
12505    #[tokio::test]
12506    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
12507        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
12508        let supervisor = Supervisor::new_for_test(
12509            Arc::new(Registry::default()),
12510            RestartPolicy::new(2, Duration::ZERO),
12511        );
12512        let runtime = supervisor.runtime_config();
12513        let spec = windowed_crash_spec("crash-loop-in-window");
12514        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12515
12516        for attempt in 1..=2 {
12517            assert!(
12518                matches!(
12519                    on_child_exit(
12520                        &spec,
12521                        runtime.restart_policy,
12522                        &supervisor.registry,
12523                        &snapshot,
12524                        &runtime.terminal_ring,
12525                        &runtime.spawn_events,
12526                        &runtime.child_roster,
12527                        crash_exit_report(attempt),
12528                    )
12529                    .await,
12530                    NextAction::Restart { schedule: _ }
12531                ),
12532                "crash {attempt} is inside the budget and must respawn"
12533            );
12534        }
12535
12536        assert!(matches!(
12537            on_child_exit(
12538                &spec,
12539                runtime.restart_policy,
12540                &supervisor.registry,
12541                &snapshot,
12542                &runtime.terminal_ring,
12543                &runtime.spawn_events,
12544                &runtime.child_roster,
12545                crash_exit_report(3),
12546            )
12547            .await,
12548            NextAction::Stop { .. }
12549        ));
12550
12551        {
12552            let state = lock_snapshot(&snapshot).unwrap();
12553            assert_eq!(state.state, ModuleState::Failed);
12554            assert_eq!(state.crash_restarts.len(), 2);
12555            assert_eq!(state.lifetime_restarts, 2);
12556        }
12557
12558        let history = runtime
12559            .terminal_ring
12560            .lock()
12561            .expect("terminal ring is not poisoned")
12562            .snapshot();
12563        let last = history
12564            .entries
12565            .last()
12566            .expect("the refused crash is retained");
12567        assert_eq!(last.disposition, TerminalDisposition::Failed);
12568        assert_eq!(
12569            last.disposition_detail.as_deref(),
12570            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
12571        );
12572
12573        let captured = crate::router::test_log::captured_logs(&logs);
12574        assert!(
12575            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
12576            "the stop must be logged with its window: {captured}"
12577        );
12578    }
12579
12580    /// The rate, stated as a test: three crashes where the first has aged past
12581    /// the window are two crashes as far as the budget is concerned, so the
12582    /// third respawn is allowed and the ring holds only the two recent ones.
12583    ///
12584    /// This is the case a lifetime counter got wrong -- and the case the daemon
12585    /// now hits routinely, since a module exits non-zero every time its
12586    /// connection to the daemon drops.
12587    #[tokio::test]
12588    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
12589        let supervisor = Supervisor::new_for_test(
12590            Arc::new(Registry::default()),
12591            RestartPolicy::new(2, Duration::ZERO),
12592        );
12593        let runtime = supervisor.runtime_config();
12594        let spec = windowed_crash_spec("crash-across-windows");
12595        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12596
12597        for attempt in 1..=2 {
12598            assert!(matches!(
12599                on_child_exit(
12600                    &spec,
12601                    runtime.restart_policy,
12602                    &supervisor.registry,
12603                    &snapshot,
12604                    &runtime.terminal_ring,
12605                    &runtime.spawn_events,
12606                    &runtime.child_roster,
12607                    crash_exit_report(attempt),
12608                )
12609                .await,
12610                NextAction::Restart { schedule: _ }
12611            ));
12612        }
12613
12614        // The oldest crash moves out of the window; nothing else about the
12615        // module changes.
12616        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12617            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
12618        })
12619        .unwrap();
12620
12621        assert!(
12622            matches!(
12623                on_child_exit(
12624                    &spec,
12625                    runtime.restart_policy,
12626                    &supervisor.registry,
12627                    &snapshot,
12628                    &runtime.terminal_ring,
12629                    &runtime.spawn_events,
12630                    &runtime.child_roster,
12631                    crash_exit_report(3),
12632                )
12633                .await,
12634                NextAction::Restart { schedule: _ }
12635            ),
12636            "a crash older than the window must not hold a budget slot"
12637        );
12638
12639        let state = lock_snapshot(&snapshot).unwrap();
12640        assert_eq!(state.state, ModuleState::Restarting);
12641        assert_eq!(
12642            state.crash_restarts.len(),
12643            2,
12644            "the aged instant is dropped and the new one takes its place"
12645        );
12646        assert_eq!(
12647            state.lifetime_restarts, 3,
12648            "the ledger counts every restart, including the ones the window forgot"
12649        );
12650    }
12651
12652    /// An operator restart hands the budget back whole, and the ledger keeps
12653    /// counting. Those are different questions -- "how close is this module to
12654    /// being stopped" and "how many times has it been replaced" -- and the
12655    /// operator action answers only the first.
12656    #[tokio::test]
12657    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
12658        let supervisor = Supervisor::new_for_test(
12659            Arc::new(Registry::default()),
12660            RestartPolicy::new(2, Duration::ZERO),
12661        );
12662        let runtime = supervisor.runtime_config();
12663        let spec = windowed_crash_spec("operator-cleared-budget");
12664        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12665
12666        for attempt in 1..=2 {
12667            assert!(matches!(
12668                on_child_exit(
12669                    &spec,
12670                    runtime.restart_policy,
12671                    &supervisor.registry,
12672                    &snapshot,
12673                    &runtime.terminal_ring,
12674                    &runtime.spawn_events,
12675                    &runtime.child_roster,
12676                    crash_exit_report(attempt),
12677                )
12678                .await,
12679                NextAction::Restart { schedule: _ }
12680            ));
12681        }
12682
12683        reset_restart_count(&snapshot, &spec.module_id).unwrap();
12684        {
12685            let state = lock_snapshot(&snapshot).unwrap();
12686            assert!(
12687                state.crash_restarts.is_empty(),
12688                "an operator restart returns the full budget"
12689            );
12690            assert_eq!(
12691                state.lifetime_restarts, 2,
12692                "clearing the budget must not unmake the crashes"
12693            );
12694        }
12695
12696        assert!(
12697            matches!(
12698                on_child_exit(
12699                    &spec,
12700                    runtime.restart_policy,
12701                    &supervisor.registry,
12702                    &snapshot,
12703                    &runtime.terminal_ring,
12704                    &runtime.spawn_events,
12705                    &runtime.child_roster,
12706                    crash_exit_report(3),
12707                )
12708                .await,
12709                NextAction::Restart { schedule: _ }
12710            ),
12711            "the cleared budget must be spendable again"
12712        );
12713        let state = lock_snapshot(&snapshot).unwrap();
12714        assert_eq!(state.crash_restarts.len(), 1);
12715        assert_eq!(state.lifetime_restarts, 3);
12716    }
12717
12718    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12719    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
12720        let severed = ProcessIdentity {
12721            pid: 41,
12722            start_time: 101,
12723        };
12724        let successor = ProcessIdentity {
12725            pid: 41,
12726            start_time: 202,
12727        };
12728        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12729        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
12730            state.pid = Some(successor.pid);
12731            state.process_start_time = Some(successor.start_time);
12732        })
12733        .unwrap();
12734        assert!(!module.record_deliberate_severance(severed).unwrap());
12735
12736        let exit_report = apply_deliberate_severance_marker(
12737            &module.inner.snapshot,
12738            Some(successor),
12739            ExitReport {
12740                kind: ExitKind::Crash,
12741                code: Some(1),
12742                signal: None,
12743                at_ms: 1,
12744            },
12745        );
12746
12747        assert_eq!(exit_report.kind, ExitKind::Crash);
12748    }
12749
12750    #[tokio::test]
12751    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
12752        let registry = Registry::default();
12753        let supervisor = Supervisor::new_for_test(
12754            Arc::new(Registry::default()),
12755            RestartPolicy::new(3, Duration::ZERO),
12756        );
12757        let runtime = supervisor.runtime_config();
12758        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12759        let spec = ModuleSpec {
12760            module_id: "drain-deliberate-severance".to_string(),
12761            program: fake_aft_stub_path(),
12762            args: Vec::new(),
12763            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12764            reserved: false,
12765            reserved_prefixes: Vec::new(),
12766            protocol: ModuleProtocol::Subc,
12767            overlap: Default::default(),
12768        };
12769        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
12770        let process = ProcessIdentity {
12771            pid: 41,
12772            start_time: 101,
12773        };
12774        child.process_identity = Some(process);
12775        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12776            state.pid = Some(process.pid);
12777            state.process_start_time = Some(process.start_time);
12778        })
12779        .unwrap();
12780        record_deliberate_severance(&snapshot, process).unwrap();
12781
12782        drain_child_to_state(
12783            &spec.module_id,
12784            spec.protocol,
12785            // The child exits on its own; no signal may change the exit this
12786            // test classifies.
12787            StopNotice::SentOverConnection,
12788            &registry,
12789            None,
12790            &snapshot,
12791            &runtime.terminal_ring,
12792            &runtime.spawn_events,
12793            child,
12794            Duration::from_secs(1),
12795            ModuleState::Stopped,
12796            Some(false),
12797        )
12798        .await
12799        .unwrap();
12800
12801        let state = lock_snapshot(&snapshot).unwrap();
12802        assert_eq!(
12803            state.last_exit.as_ref().map(|exit| exit.kind),
12804            Some(ExitKind::DeliberateSeverance)
12805        );
12806        assert_eq!(state.lifetime_restarts, 1);
12807        assert_eq!(state.crash_restarts.len(), 0);
12808        drop(state);
12809        let history = runtime.terminal_ring.lock().unwrap().snapshot();
12810        assert_eq!(
12811            history.entries[0].exit_kind,
12812            subc_control::TerminalExitKind::DeliberateSeverance
12813        );
12814    }
12815
12816    #[tokio::test]
12817    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
12818        let registry = Registry::default();
12819        let supervisor = Supervisor::new_for_test(
12820            Arc::new(Registry::default()),
12821            RestartPolicy::new(3, Duration::ZERO),
12822        );
12823        let runtime = supervisor.runtime_config();
12824        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12825        let spec = ModuleSpec {
12826            module_id: "ordinary-drain".to_string(),
12827            program: fake_aft_stub_path(),
12828            args: Vec::new(),
12829            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12830            reserved: false,
12831            reserved_prefixes: Vec::new(),
12832            protocol: ModuleProtocol::Subc,
12833            overlap: Default::default(),
12834        };
12835        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
12836
12837        drain_child_to_state(
12838            &spec.module_id,
12839            spec.protocol,
12840            // The child exits on its own; no signal may change the exit this
12841            // test classifies.
12842            StopNotice::SentOverConnection,
12843            &registry,
12844            None,
12845            &snapshot,
12846            &runtime.terminal_ring,
12847            &runtime.spawn_events,
12848            child,
12849            Duration::from_secs(1),
12850            ModuleState::Stopped,
12851            Some(false),
12852        )
12853        .await
12854        .unwrap();
12855
12856        let state = lock_snapshot(&snapshot).unwrap();
12857        assert_eq!(
12858            state.last_exit.as_ref().map(|exit| exit.kind),
12859            Some(ExitKind::Crash)
12860        );
12861        assert_eq!(state.lifetime_restarts, 0);
12862        assert_eq!(state.crash_restarts.len(), 0);
12863    }
12864
12865    #[test]
12866    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
12867        // The server's generic fatal-routing branch only knows that the
12868        // connection failed; it does not know that the daemon deliberately
12869        // initiated a process-killing severance. Keep this seam explicit so a
12870        // future connection error path cannot silently reintroduce the stale
12871        // exemption that mislabels a later genuine crash.
12872        assert!(!include_str!("server.rs")
12873            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
12874    }
12875
12876    /// The `route.closed` `drained` value must be the quiescence wait's own
12877    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
12878    /// measurement at all and `false` is the one honest constant. This is the exact
12879    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
12880    /// on every return path, including the one that used to return early via `?`
12881    /// with `route.closing` already sent and no `route.closed` ever following.
12882    #[test]
12883    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
12884        assert!(drained_after_quiescence_wait(&Ok(true)));
12885        assert!(!drained_after_quiescence_wait(&Ok(false)));
12886        assert!(!drained_after_quiescence_wait(&Err(
12887            SuperviseError::StatePoisoned { module_id: None }
12888        )));
12889    }
12890
12891    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
12892    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
12893    /// already reaped out-of-band) still leaves a terminal record rather than none
12894    /// at all. Triggering the real `wait()` I/O error from an integration test would
12895    /// need a genuine already-reaped-child race, which is OS-specific and not
12896    /// something this suite attempts elsewhere; this test instead verifies the
12897    /// record produced for that arm end-to-end through the real `TerminalRing`, and
12898    /// the call site itself is verified by inspection to sit in that exact arm.
12899    #[test]
12900    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
12901        let ring = Arc::new(Mutex::new(TerminalRing::new(
12902            TerminalRingConfig::default(),
12903            0,
12904        )));
12905        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
12906
12907        let snapshot = ring.lock().unwrap().snapshot();
12908        assert_eq!(snapshot.entries.len(), 1);
12909        let entry = &snapshot.entries[0];
12910        assert_eq!(entry.exit_code, None);
12911        assert_eq!(entry.exit_signal, None);
12912        assert_eq!(entry.disposition, TerminalDisposition::Failed);
12913    }
12914
12915    #[test]
12916    fn wait_error_exit_path_preserves_spawn_event_density() {
12917        let feed = super::SpawnEventFeed::default();
12918        feed.configure_incarnation("wait-error-density".to_string());
12919        feed.emit_spawned("wait-error", 41, 1);
12920        let ring = Arc::new(Mutex::new(TerminalRing::new(
12921            TerminalRingConfig::default(),
12922            0,
12923        )));
12924
12925        record_wait_error_terminal("wait-error", &ring, &feed);
12926        feed.emit_spawned("after-wait-error", 42, 2);
12927
12928        let state = feed.0.lock().unwrap();
12929        let sequences = state
12930            .events
12931            .iter()
12932            .map(|event| event.cursor.seq)
12933            .collect::<Vec<_>>();
12934        assert_eq!(sequences, vec![1, 2, 3]);
12935        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
12936        assert_eq!(state.events[1].exit_code, None);
12937        assert_eq!(state.events[1].exit_signal, None);
12938    }
12939
12940    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
12941    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
12942    /// not a clean exit it never actually observed.
12943    #[test]
12944    fn wait_error_exit_report_is_classified_as_a_crash() {
12945        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
12946    }
12947}
12948
12949#[cfg(all(test, unix))]
12950mod health_event_tests {
12951    use super::*;
12952    use cortexkit_test_support::ScratchDir;
12953    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
12954
12955    struct Actor {
12956        module: SupervisedModule,
12957        registry: Arc<Registry>,
12958        forwarding: Arc<ForwardingTable>,
12959        spec: ModuleSpec,
12960        health: HealthConfig,
12961        rx: mpsc::Receiver<crate::router::OutboundFrame>,
12962        _home: ScratchDir,
12963    }
12964
12965    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
12966        // OS exec and socket readiness are real I/O. Stay runnable while waiting
12967        // for an observed condition, rather than letting paused time auto-advance
12968        // deadlines or assuming a fixed number of yields finishes the work.
12969        let deadline = std::time::Instant::now() + Duration::from_secs(10);
12970        let tick = Instant::now();
12971        loop {
12972            if let Some(value) = observe() {
12973                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
12974                return value;
12975            }
12976            assert!(
12977                std::time::Instant::now() < deadline,
12978                "timed out waiting for {reason}"
12979            );
12980            tokio::task::yield_now().await;
12981        }
12982    }
12983
12984    impl Actor {
12985        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
12986            let home = ScratchDir::new("health-events");
12987            let registry = Arc::new(Registry::default());
12988            let forwarding = Arc::new(ForwardingTable::default());
12989            let health = HealthConfig {
12990                cadence,
12991                deadline: Duration::from_secs(3600),
12992                ..HealthConfig::default()
12993            };
12994            let supervisor =
12995                Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
12996                    .with_forwarding(forwarding.clone())
12997                    .with_health_config(health.clone());
12998            let spec = ModuleSpec {
12999                module_id: "event-health-child".into(),
13000                program: PathBuf::from("/bin/sh"),
13001                // The process stays alive but has no real bus peer. Registrations
13002                // below exercise the same registry writes as accepted HELLOs.
13003                args: vec!["-c".into(), "exec sleep 600".into()],
13004                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13005                    .into_iter()
13006                    .map(|key| (key.into(), home.path().display().to_string()))
13007                    .collect(),
13008                reserved: false,
13009                reserved_prefixes: vec![],
13010                protocol,
13011                overlap: Default::default(),
13012            };
13013            let module = supervisor.spawn(spec.clone()).unwrap();
13014            let (_, rx) = mpsc::channel(8);
13015            let actor = Self {
13016                module,
13017                registry,
13018                forwarding,
13019                spec,
13020                health,
13021                rx,
13022                _home: home,
13023            };
13024            actor
13025                .wait_for_select(|checkpoint| checkpoint.generation > 0)
13026                .await;
13027            actor
13028        }
13029
13030        fn turns(&self) -> u64 {
13031            lock_snapshot(&self.module.inner.snapshot)
13032                .unwrap()
13033                .actor_turns
13034        }
13035
13036        async fn wait_for_select(
13037            &self,
13038            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13039        ) -> ActorSelectCheckpoint {
13040            wait_for("actor select checkpoint", || {
13041                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13042                state.actor_select.filter(|checkpoint| {
13043                    checkpoint.generation == state.spawn_generation
13044                        && state.state == ModuleState::Running
13045                        && state.process_alive
13046                        && state.reported_pid().is_some()
13047                        && expected(checkpoint)
13048                })
13049            })
13050            .await
13051        }
13052
13053        fn hello(&mut self, connection: u64, health: bool) {
13054            let manifest =
13055                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13056                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13057                    .build();
13058            let connection = ConnectionId::new(connection);
13059            let (tx, rx) = mpsc::channel(8);
13060            self.rx = rx;
13061            self.forwarding
13062                .register_module_connection(
13063                    connection,
13064                    self.spec.module_id.clone(),
13065                    subc_protocol::PROTOCOL_VERSION,
13066                    Concurrency::ModuleManaged,
13067                    FrameSink::new(tx),
13068                )
13069                .unwrap();
13070            self.registry
13071                .register_with_control_ops(
13072                    manifest,
13073                    subc_protocol::PROTOCOL_VERSION,
13074                    connection,
13075                    if health {
13076                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13077                    } else {
13078                        vec![]
13079                    },
13080                )
13081                .unwrap();
13082        }
13083
13084        async fn probe(&mut self) -> crate::router::OutboundFrame {
13085            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13086                Ok(frame) => Some(frame),
13087                Err(mpsc::error::TryRecvError::Empty) => None,
13088                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13089            })
13090            .await;
13091            let body: Value = serde_json::from_slice(&frame.body).unwrap();
13092            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13093            frame
13094        }
13095
13096        fn no_probe(&mut self) {
13097            assert!(matches!(
13098                self.rx.try_recv(),
13099                Err(mpsc::error::TryRecvError::Empty)
13100            ));
13101        }
13102
13103        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13104            self.forwarding
13105                .complete_module_control_rpc(
13106                    ConnectionId::new(connection),
13107                    frame.header.corr,
13108                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13109                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13110                        status: HealthStatus::Ok,
13111                        detail: None,
13112                        metrics: None,
13113                    }),
13114                )
13115                .unwrap();
13116        }
13117    }
13118
13119    #[tokio::test(start_paused = true)]
13120    async fn no_health_children_park_for_sixty_seconds() {
13121        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13122        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13123        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13124        unadvertised.hello(1, false);
13125        // Include any one-off registration wake in the budget; receiving a
13126        // no-health HELLO must not turn a parked child into a polling child.
13127        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
13128        // a single 60 s jump would only observe one expired timer.
13129        for _ in 0..6000 {
13130            tokio::time::advance(Duration::from_millis(10)).await;
13131            tokio::task::yield_now().await;
13132        }
13133        for actor in [&none, &unregistered, &unadvertised] {
13134            assert!(
13135                actor.turns() <= 5,
13136                "no-health actor woke {} times",
13137                actor.turns()
13138            );
13139            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13140        }
13141    }
13142
13143    #[tokio::test(start_paused = true)]
13144    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13145        // With a zero cadence the first probe is due at once, so this measures
13146        // only how fast the registration notification starts health probing,
13147        // not the normal 30-33 s wait before a first probe (checked below).
13148        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13149        let tick = Instant::now();
13150        actor.hello(1, true);
13151        actor.probe().await;
13152        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13153    }
13154
13155    #[tokio::test(start_paused = true)]
13156    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13157        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13158        let hello_tick = Instant::now();
13159        actor.hello(1, true);
13160        let armed = actor
13161            .wait_for_select(|checkpoint| {
13162                checkpoint.registered_connection == Some(ConnectionId::new(1))
13163                    && checkpoint.next_probe_at.is_some()
13164            })
13165            .await;
13166        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13167        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13168        let first_deadline = hello_tick + delay;
13169        assert_eq!(armed.next_probe_at, Some(first_deadline));
13170        tokio::time::advance(Duration::from_secs(29)).await;
13171        let turns = actor.turns();
13172        actor
13173            .registry
13174            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13175            .unwrap();
13176        let updated = actor
13177            .wait_for_select(|checkpoint| checkpoint.turn > turns)
13178            .await;
13179        assert_eq!(
13180            updated.next_probe_at,
13181            Some(first_deadline),
13182            "catalog update must not reset cadence"
13183        );
13184        actor.no_probe();
13185        tokio::time::advance(first_deadline - Instant::now()).await;
13186        let frame = actor.probe().await;
13187        actor.answer(1, frame);
13188        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13189        let next_deadline = Instant::now() + next;
13190        let rearmed = actor
13191            .wait_for_select(|checkpoint| {
13192                checkpoint
13193                    .next_probe_at
13194                    .is_some_and(|deadline| deadline > first_deadline)
13195            })
13196            .await;
13197        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
13198        tokio::time::advance(next - Duration::from_nanos(1)).await;
13199        actor.no_probe();
13200        tokio::time::advance(Duration::from_nanos(1)).await;
13201        actor.probe().await;
13202    }
13203
13204    #[tokio::test(start_paused = true)]
13205    async fn disconnect_disarms_and_reconnect_rearms_health() {
13206        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13207        actor.hello(1, true);
13208        actor
13209            .wait_for_select(|checkpoint| {
13210                checkpoint.registered_connection == Some(ConnectionId::new(1))
13211                    && checkpoint.next_probe_at.is_some()
13212            })
13213            .await;
13214        let turns = actor.turns();
13215        let tick = Instant::now();
13216        actor
13217            .registry
13218            .deregister_connection(ConnectionId::new(1))
13219            .unwrap();
13220        let disarmed = actor
13221            .wait_for_select(|checkpoint| {
13222                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
13223            })
13224            .await;
13225        assert_eq!(disarmed.next_probe_at, None);
13226        assert_eq!(disarmed.wake_after, None);
13227        assert_eq!(
13228            Instant::now(),
13229            tick,
13230            "disconnect must disarm in the notification tick"
13231        );
13232        tokio::time::advance(Duration::from_secs(60)).await;
13233        actor.no_probe();
13234        let reconnect_tick = Instant::now();
13235        actor.hello(2, true);
13236        let rearmed = actor
13237            .wait_for_select(|checkpoint| {
13238                checkpoint.registered_connection == Some(ConnectionId::new(2))
13239                    && checkpoint.next_probe_at.is_some()
13240            })
13241            .await;
13242        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13243        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
13244        tokio::time::advance(delay).await;
13245        actor.probe().await;
13246    }
13247
13248    #[tokio::test(start_paused = true)]
13249    async fn reregistration_without_health_disarms_the_old_schedule() {
13250        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13251        actor.hello(1, true);
13252        actor
13253            .wait_for_select(|checkpoint| {
13254                checkpoint.registered_connection == Some(ConnectionId::new(1))
13255                    && checkpoint.next_probe_at.is_some()
13256            })
13257            .await;
13258        actor
13259            .registry
13260            .deregister_connection(ConnectionId::new(1))
13261            .unwrap();
13262        actor.hello(2, false);
13263        let disarmed = actor
13264            .wait_for_select(|checkpoint| {
13265                checkpoint.registered_connection == Some(ConnectionId::new(2))
13266            })
13267            .await;
13268        assert_eq!(disarmed.next_probe_at, None);
13269        assert_eq!(disarmed.wake_after, None);
13270        tokio::time::advance(Duration::from_secs(60)).await;
13271        actor.no_probe();
13272    }
13273
13274    #[tokio::test(start_paused = true)]
13275    async fn restart_waits_for_its_own_hello_before_rearming_health() {
13276        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13277        actor.hello(1, true);
13278        let armed = actor
13279            .wait_for_select(|checkpoint| {
13280                checkpoint.registered_connection == Some(ConnectionId::new(1))
13281                    && checkpoint.next_probe_at.is_some()
13282            })
13283            .await;
13284        // Simulate the old process's connection closing before the restart
13285        // tears the process down. The replacement process has started but has
13286        // not sent its HELLO, so it is not registered.
13287        actor
13288            .registry
13289            .deregister_connection(ConnectionId::new(1))
13290            .unwrap();
13291        actor.module.restart(Some(0)).await.unwrap();
13292        let parked = actor
13293            .wait_for_select(|checkpoint| {
13294                checkpoint.generation > armed.generation
13295                    && checkpoint.registered_connection.is_none()
13296                    && checkpoint.next_probe_at.is_none()
13297            })
13298            .await;
13299        assert_eq!(parked.wake_after, None);
13300        actor.rx.close();
13301        // Drain the stop notice the restart queued for the old process's connection,
13302        // so it isn't mistaken for traffic to the replacement.
13303        while actor.rx.try_recv().is_ok() {}
13304        let turns = actor.turns();
13305        tokio::time::advance(Duration::from_secs(60)).await;
13306        let after = actor.turns();
13307        eprintln!(
13308            "restart park turns: before={turns} after={after} delta={}",
13309            after - turns
13310        );
13311        assert_eq!(after, turns, "replacement must park before HELLO");
13312        let hello_tick = Instant::now();
13313        actor.hello(2, true);
13314        let rearmed = actor
13315            .wait_for_select(|checkpoint| {
13316                checkpoint.registered_connection == Some(ConnectionId::new(2))
13317                    && checkpoint.next_probe_at.is_some()
13318            })
13319            .await;
13320        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13321        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
13322        tokio::time::advance(delay).await;
13323        actor.probe().await;
13324    }
13325
13326    #[tokio::test(start_paused = true)]
13327    async fn rescan_adds_and_removes_http_health_without_polling() {
13328        use tokio::{
13329            io::{AsyncReadExt, AsyncWriteExt},
13330            net::TcpListener,
13331        };
13332        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13333        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
13334        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
13335        // One millisecond separates successive probes without a polling delay.
13336        actor.health.cadence = Duration::from_millis(1);
13337        let (started_tx, mut started_rx) = mpsc::channel(8);
13338        let peer = tokio::spawn(async move {
13339            loop {
13340                let (mut socket, _) = listener.accept().await.unwrap();
13341                let mut request = [0; 1024];
13342                assert!(socket.read(&mut request).await.unwrap() > 0);
13343                started_tx.send(()).await.unwrap();
13344                socket
13345                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
13346                    .await
13347                    .unwrap();
13348            }
13349        });
13350        let tick = Instant::now();
13351        let turns = actor.turns();
13352        actor
13353            .module
13354            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13355            .await
13356            .unwrap();
13357        let armed = actor
13358            .wait_for_select(|checkpoint| {
13359                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
13360            })
13361            .await;
13362        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
13363        assert_eq!(Instant::now(), tick);
13364        tokio::time::advance(Duration::from_millis(1)).await;
13365        let mut started = false;
13366        wait_for("successful HTTP probe", || {
13367            started |= started_rx.try_recv().is_ok();
13368            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
13369                .then_some(())
13370        })
13371        .await;
13372        assert!(started, "added HTTP check must start at its first deadline");
13373        assert_eq!(
13374            actor.module.status().unwrap().health.status,
13375            SupervisorHealthStatus::Ok
13376        );
13377        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
13378        actor.health.http = None;
13379        let turns = actor.turns();
13380        actor
13381            .module
13382            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13383            .await
13384            .unwrap();
13385        let parked = actor
13386            .wait_for_select(|checkpoint| {
13387                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
13388            })
13389            .await;
13390        assert_eq!(parked.wake_after, None);
13391        let turns = actor.turns();
13392        tokio::time::advance(Duration::from_secs(60)).await;
13393        assert_eq!(
13394            actor.turns(),
13395            turns,
13396            "removed HTTP check must park the actor"
13397        );
13398        assert!(
13399            started_rx.try_recv().is_err(),
13400            "removed HTTP check must stay disarmed"
13401        );
13402        assert_eq!(
13403            actor.module.status().unwrap().health,
13404            ModuleHealthStatus::default()
13405        );
13406        peer.abort();
13407    }
13408}
13409
13410#[cfg(test)]
13411mod health_evidence_tests {
13412    use super::{HealthProbeError, HealthProbeEvidence};
13413    use std::collections::HashSet;
13414
13415    /// The evidential asymmetry, asserted rather than described.
13416    ///
13417    /// Exactly ONE observation is proof a module cannot serve, and the one that
13418    /// fires under CPU starvation is not it. Before the split, all fifteen
13419    /// construction sites collapsed into a single String, so a timeout carried the
13420    /// same weight as a dead lane -- which is how a healthy module was restarted
13421    /// three times in one day.
13422    #[test]
13423    fn only_a_dead_lane_is_proof_of_death() {
13424        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
13425        // Three non-proof classes, each for a different reason: silence is
13426        // consistent with health, a bad answer proves the module ALIVE, and a
13427        // daemon-side fault never reached the module at all.
13428        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
13429        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
13430        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
13431    }
13432
13433    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
13434    ///
13435    /// A shared label renders two different observations identically in the line an
13436    /// operator reads after an unexplained restart -- the exact confusion this
13437    /// change removes.
13438    #[test]
13439    fn every_evidence_class_has_a_distinct_label() {
13440        let labels = [
13441            HealthProbeError::lane_dead("").label(),
13442            HealthProbeError::no_answer("").label(),
13443            HealthProbeError::bad_answer("").label(),
13444            HealthProbeError::misconfigured("").label(),
13445        ];
13446        let unique: HashSet<_> = labels.iter().collect();
13447        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
13448    }
13449
13450    /// The class is additional information, not a replacement.
13451    ///
13452    /// An operator needs both "this was silence" and the specific text saying how
13453    /// long we waited; a classification that swallowed the message would trade one
13454    /// missing distinction for another.
13455    #[test]
13456    fn classification_preserves_the_original_message() {
13457        let err = HealthProbeError::no_answer("module did not answer within 5s");
13458        assert_eq!(err.to_string(), "module did not answer within 5s");
13459        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13460    }
13461}
13462
13463#[cfg(test)]
13464mod health_tombstone_tests {
13465    use std::{path::PathBuf, sync::Arc, time::Duration};
13466
13467    use subc_protocol::{
13468        manifest::Concurrency,
13469        session::{HealthStatus, ModuleControlResponse},
13470    };
13471    use tokio::sync::mpsc;
13472
13473    use super::{
13474        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
13475        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
13476    };
13477    use crate::{
13478        control::ControlHandler,
13479        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
13480        registry::{ConnectionId, Registry},
13481        router::FrameSink,
13482    };
13483
13484    struct ProbeHarness {
13485        spec: ModuleSpec,
13486        runtime: SupervisorRuntimeConfig,
13487        forwarding: Arc<ForwardingTable>,
13488        module_connection: ConnectionId,
13489        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13490        handler: ControlHandler,
13491        module: super::SupervisedModule,
13492    }
13493
13494    fn probe_harness() -> ProbeHarness {
13495        let registry = Arc::new(Registry::default());
13496        let forwarding = Arc::new(ForwardingTable::default());
13497        let supervisor_handle = super::SupervisorHandle::new();
13498        let health = HealthConfig {
13499            http: None,
13500            cadence: Duration::from_secs(30),
13501            deadline: Duration::from_secs(5),
13502            failure_threshold: 3,
13503            on_degraded: HealthAction::Report,
13504            on_failing: HealthAction::Report,
13505            critical: false,
13506        };
13507        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
13508            .with_forwarding(Arc::clone(&forwarding))
13509            .with_handle(supervisor_handle.clone())
13510            .with_health_config(health);
13511        let spec = ModuleSpec {
13512            module_id: "late-health-module".to_string(),
13513            program: PathBuf::from("disabled-module"),
13514            args: Vec::new(),
13515            env: Vec::new(),
13516            reserved: false,
13517            reserved_prefixes: Vec::new(),
13518            protocol: ModuleProtocol::Subc,
13519            overlap: Default::default(),
13520        };
13521        let module = supervisor
13522            .supervise_configured(spec.clone(), false)
13523            .unwrap();
13524        let runtime = supervisor.runtime_config();
13525        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
13526            .with_supervisor(supervisor_handle);
13527        let module_connection = ConnectionId::new(700);
13528        let (module_tx, module_rx) = mpsc::channel(8);
13529        forwarding
13530            .register_module_connection(
13531                module_connection,
13532                spec.module_id.clone(),
13533                subc_protocol::PROTOCOL_VERSION,
13534                Concurrency::ModuleManaged,
13535                FrameSink::new(module_tx),
13536            )
13537            .unwrap();
13538
13539        ProbeHarness {
13540            spec,
13541            runtime,
13542            forwarding,
13543            module_connection,
13544            module_rx,
13545            handler,
13546            module,
13547        }
13548    }
13549
13550    async fn finish_after(
13551        harness: &mut ProbeHarness,
13552        stall: Duration,
13553    ) -> ModuleControlRpcCompletion {
13554        assert!(stall > harness.runtime.health.deadline);
13555        let deadline = harness.runtime.health.deadline;
13556        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
13557        let answer = async {
13558            let frame = harness.module_rx.recv().await.expect("health.check frame");
13559            tokio::time::advance(deadline).await;
13560            tokio::task::yield_now().await;
13561            tokio::time::advance(stall - deadline).await;
13562            harness
13563                .forwarding
13564                .complete_module_control_rpc(
13565                    harness.module_connection,
13566                    frame.header.corr,
13567                    Some("health.check"),
13568                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13569                        status: HealthStatus::Ok,
13570                        detail: None,
13571                        metrics: None,
13572                    }),
13573                )
13574                .unwrap()
13575        };
13576        let (probe_result, completion) = tokio::join!(probe, answer);
13577        let err = probe_result.expect_err("probe must miss its deadline");
13578        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13579        completion
13580    }
13581
13582    async fn time_out_without_answer(harness: &mut ProbeHarness) {
13583        let deadline = harness.runtime.health.deadline;
13584        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
13585        let exhaust_deadline = async {
13586            let _frame = harness.module_rx.recv().await.expect("health.check frame");
13587            tokio::time::advance(deadline).await;
13588            tokio::task::yield_now().await;
13589        };
13590        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
13591        let err = probe_result.expect_err("probe must miss its deadline");
13592        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13593    }
13594
13595    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
13596        let registry = Arc::clone(&harness.module.inner.registry);
13597        let snapshot = Arc::clone(&harness.module.inner.snapshot);
13598        let process_liveness = super::SupervisorProcessLiveness::default();
13599        let mut child = None;
13600        let cycle = super::run_health_probe_cycle(
13601            &harness.spec,
13602            &harness.runtime,
13603            &registry,
13604            &process_liveness,
13605            &snapshot,
13606            &mut child,
13607        );
13608        let peer = async {
13609            let frame = harness.module_rx.recv().await.expect("health.check frame");
13610            if answer {
13611                harness
13612                    .forwarding
13613                    .complete_module_control_rpc(
13614                        harness.module_connection,
13615                        frame.header.corr,
13616                        Some("health.check"),
13617                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13618                            status: HealthStatus::Ok,
13619                            detail: None,
13620                            metrics: Some(serde_json::json!({"ready": true})),
13621                        }),
13622                    )
13623                    .unwrap();
13624            } else {
13625                tokio::time::advance(harness.runtime.health.deadline).await;
13626                tokio::task::yield_now().await;
13627            }
13628        };
13629        tokio::join!(cycle, peer);
13630    }
13631
13632    #[tokio::test(start_paused = true)]
13633    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
13634        let mut harness = probe_harness();
13635        // Drive the probe cycle directly with an in-memory wire peer. Stop the
13636        // disabled module's monitor so only this test owns lifecycle transitions;
13637        // no OS process is launched, and a restart is observed at scheduling.
13638        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
13639        monitor.abort();
13640        let _ = monitor.await;
13641        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
13642            state.enabled = true;
13643            state.state = super::ModuleState::Running;
13644            state.process_alive = true;
13645        })
13646        .unwrap();
13647
13648        run_probe_cycle(&mut harness, true).await;
13649        assert_eq!(
13650            harness.module.status().unwrap().health.status,
13651            super::SupervisorHealthStatus::Ok
13652        );
13653
13654        for failures in 1..harness.runtime.health.failure_threshold {
13655            run_probe_cycle(&mut harness, false).await;
13656            let status = harness.module.status().unwrap();
13657            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
13658            assert_eq!(status.health.consecutive_failures, failures);
13659            assert!(status.health.last_probe_ms.is_some());
13660            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
13661            assert!(status.health.metrics.is_none());
13662            assert_eq!(status.state, super::ModuleState::Running);
13663            assert!(status.process_alive);
13664            assert_eq!(status.restart_count, 0);
13665            assert_eq!(status.lifetime_restarts, 0);
13666            assert!(status.health.last_action.is_none());
13667            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
13668        }
13669
13670        run_probe_cycle(&mut harness, true).await;
13671        let recovered = harness.module.status().unwrap();
13672        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
13673        assert_eq!(recovered.health.consecutive_failures, 0);
13674        assert!(recovered.health.detail.is_none());
13675        assert_eq!(
13676            recovered.health.metrics,
13677            Some(serde_json::json!({"ready": true}))
13678        );
13679        assert_eq!(recovered.lifetime_restarts, 0);
13680
13681        for failures in 1..=harness.runtime.health.failure_threshold {
13682            run_probe_cycle(&mut harness, false).await;
13683            let status = harness.module.status().unwrap();
13684            assert_eq!(status.health.consecutive_failures, failures);
13685            if failures < harness.runtime.health.failure_threshold {
13686                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
13687                assert_eq!(status.state, super::ModuleState::Running);
13688                assert_eq!(status.lifetime_restarts, 0);
13689                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
13690            } else {
13691                assert_eq!(
13692                    status.health.status,
13693                    super::SupervisorHealthStatus::Unresponsive
13694                );
13695                assert_eq!(status.state, super::ModuleState::Restarting);
13696                assert_eq!(status.restart_count, 1);
13697                assert_eq!(status.lifetime_restarts, 1);
13698                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
13699                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
13700            }
13701        }
13702    }
13703
13704    #[tokio::test(start_paused = true)]
13705    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
13706        let mut harness = probe_harness();
13707
13708        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
13709        let first_latency = match &first {
13710            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
13711            other => panic!("late answer was not retained: {other:?}"),
13712        };
13713        assert!(harness.handler.observe_module_control_completion(first));
13714
13715        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
13716        let second_latency = match &second {
13717            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
13718            other => panic!("late answer was not retained: {other:?}"),
13719        };
13720        assert!(harness.handler.observe_module_control_completion(second));
13721
13722        assert_eq!(first_latency, Duration::from_secs(8));
13723        assert_eq!(
13724            second_latency - first_latency,
13725            Duration::from_secs(3),
13726            "latency must grow linearly with the additional stall"
13727        );
13728        let health = harness.module.status().unwrap().health;
13729        assert_eq!(health.late_answer_count, 2);
13730        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
13731    }
13732
13733    /// A module that answers every probe late must never march to the kill
13734    /// threshold: the late answer proves it is alive, so it must clear the miss
13735    /// streak the timeout recorded. Without the reset, a CPU-starved module
13736    /// that serves every probe seconds past the deadline accumulates
13737    /// `consecutive_failures` to the threshold and is killed — the exact
13738    /// sequence from the 2026-08-14 aft disable, where the daemon logged
13739    /// "proves the module is alive" five times while counting five misses.
13740    #[tokio::test(start_paused = true)]
13741    async fn late_answer_clears_the_consecutive_failure_streak() {
13742        let mut harness = probe_harness();
13743
13744        // Timeout recorded first: the probe path saw no answer in time.
13745        time_out_without_answer(&mut harness).await;
13746        harness
13747            .module
13748            .record_health_probe_failure_for_test("[no-answer] test miss")
13749            .unwrap();
13750        assert_eq!(
13751            harness.module.status().unwrap().health.consecutive_failures,
13752            1,
13753            "precondition: the miss must be on the streak before the late answer"
13754        );
13755
13756        // The stalled reply then lands: proof of life.
13757        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
13758        assert!(matches!(
13759            late,
13760            ModuleControlRpcCompletion::LateHealthAnswer { .. }
13761        ));
13762        assert!(harness.handler.observe_module_control_completion(late));
13763
13764        let health = harness.module.status().unwrap().health;
13765        assert_eq!(
13766            health.consecutive_failures, 0,
13767            "a late answer is an answer: the streak must reset"
13768        );
13769        assert_eq!(health.late_answer_count, 1);
13770    }
13771
13772    #[tokio::test(start_paused = true)]
13773    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
13774        let mut harness = probe_harness();
13775
13776        for _ in 0..20 {
13777            time_out_without_answer(&mut harness).await;
13778            assert_eq!(
13779                harness.forwarding.health_probe_tombstone_count().unwrap(),
13780                1
13781            );
13782        }
13783    }
13784}
13785
13786#[cfg(test)]
13787mod child_env_tests {
13788    #[cfg(windows)]
13789    use super::{
13790        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
13791    };
13792    #[cfg(windows)]
13793    use std::sync::{Arc, OnceLock};
13794    #[cfg(windows)]
13795    #[test]
13796    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
13797        let spec = spec(Vec::new());
13798        let mut command = Command::new("unused");
13799        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
13800            .unwrap()
13801            .unwrap();
13802        let name = command
13803            .as_std()
13804            .get_envs()
13805            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
13806            .and_then(|(_, value)| value)
13807            .unwrap();
13808        assert_eq!(name, OsStr::new(handoff.name()));
13809        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
13810        assert!(command.as_std().get_envs().any(|(key, value)| key
13811            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
13812            && value == Some(OsStr::new("env"))));
13813        assert!(command
13814            .as_std()
13815            .get_envs()
13816            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
13817    }
13818
13819    #[cfg(windows)]
13820    #[test]
13821    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
13822        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
13823        snapshot.process_alive = true;
13824        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
13825        assert_eq!(
13826            observed_launch_nonce_source(&snapshot).as_deref(),
13827            Some("env")
13828        );
13829        let consumed = Arc::new(OnceLock::new());
13830        snapshot.pipe_consumed = Some(consumed.clone());
13831        assert_eq!(
13832            observed_launch_nonce_source(&snapshot).as_deref(),
13833            Some("env")
13834        );
13835        consumed.set(()).unwrap();
13836        assert_eq!(
13837            observed_launch_nonce_source(&snapshot).as_deref(),
13838            Some("pipe")
13839        );
13840        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
13841        assert_eq!(
13842            observed_launch_nonce_source(&snapshot).as_deref(),
13843            Some("env")
13844        );
13845        clear_current_process_facts(&mut snapshot);
13846        assert_eq!(observed_launch_nonce_source(&snapshot), None);
13847    }
13848    use super::{
13849        apply_child_env, apply_spawn_role, apply_wire_spawn_args, ModuleProtocol, ModuleSpec,
13850        SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG, SUBC_LAUNCH_NONCE_ENV,
13851        SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
13852    };
13853    use std::{ffi::OsStr, path::PathBuf};
13854    use tokio::process::Command;
13855
13856    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
13857        ModuleSpec {
13858            module_id: "env-plan".to_string(),
13859            program: PathBuf::from("/nonexistent"),
13860            args: Vec::new(),
13861            env,
13862            reserved: false,
13863            reserved_prefixes: Vec::new(),
13864            protocol: ModuleProtocol::Subc,
13865            overlap: Default::default(),
13866        }
13867    }
13868
13869    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
13870    /// one still gets its own.
13871    ///
13872    /// This is the narrow goal `env_clear()` was reached for, and the reason the
13873    /// fix is `env_remove` rather than deleting the line: an operator's ambient
13874    /// filter silently becoming an unconfigured module's log level is a real
13875    /// defect, just a much smaller one than clearing the environment.
13876    ///
13877    /// Asserted on the command plan rather than a spawned child because proving
13878    /// the ABSENCE of an inherited variable needs the parent's environment
13879    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
13880    /// removal as `(key, None)`, which is exactly the distinction wanted: not
13881    /// "absent because nobody set it" but "explicitly unset for the child".
13882    #[test]
13883    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
13884        let mut command = Command::new("/nonexistent");
13885        apply_child_env(&mut command, &spec(Vec::new()));
13886        let removed = command
13887            .as_std()
13888            .get_envs()
13889            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
13890        assert!(
13891            removed,
13892            "ambient CK_LOG must be explicitly removed for an unconfigured module"
13893        );
13894
13895        let mut configured = Command::new("/nonexistent");
13896        apply_child_env(
13897            &mut configured,
13898            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
13899        );
13900        let effective = configured
13901            .as_std()
13902            .get_envs()
13903            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
13904            .last()
13905            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
13906        assert_eq!(
13907            effective,
13908            Some(Some("debug".to_string())),
13909            "a module's configured CK_LOG must survive the ambient removal"
13910        );
13911    }
13912
13913    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
13914    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
13915    /// the same reason as the CK_LOG test above.
13916    ///
13917    /// The argument is the load-bearing half: a stock binary exits on an
13918    /// unknown flag before it listens, so with `--subc` appended the mode
13919    /// could not supervise the one process it exists for. Found by the first
13920    /// conformance run (nats-server: `flag provided but not defined: -subc`).
13921    #[test]
13922    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
13923        let connection_file = std::path::Path::new("/run/subc-connection.json");
13924        let handle = SupervisorHandle::new();
13925
13926        let mut none_spec = spec(Vec::new());
13927        none_spec.protocol = ModuleProtocol::None;
13928        let mut none = Command::new("/nonexistent");
13929        let none_handoff =
13930            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
13931                .expect("protocol-none spawn args apply");
13932        assert!(
13933            none_handoff.is_none(),
13934            "protocol:none spawn must not receive a nonce descriptor"
13935        );
13936        #[cfg(windows)]
13937        assert!(!none.as_std().get_envs().any(|(key, value)| key
13938            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
13939            && value.is_some()));
13940        assert!(
13941            !none.as_std().get_envs().any(|(key, value)| key
13942                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
13943                && value.is_some()),
13944            "protocol:none spawn must not name a nonce descriptor"
13945        );
13946        let none_args: Vec<String> = none
13947            .as_std()
13948            .get_args()
13949            .map(|a| a.to_string_lossy().into_owned())
13950            .collect();
13951        assert!(
13952            !none_args.iter().any(|a| a == SUBC_ARG),
13953            "protocol:none argv must not carry --subc; got {none_args:?}"
13954        );
13955        let none_has_nonce = none
13956            .as_std()
13957            .get_envs()
13958            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
13959        assert!(
13960            !none_has_nonce,
13961            "protocol:none spawn must not receive a launch nonce"
13962        );
13963        let none_has_module_id = none
13964            .as_std()
13965            .get_envs()
13966            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
13967        assert!(
13968            none_has_module_id,
13969            "SUBC_MODULE_ID is inert and stays on every path"
13970        );
13971        assert!(
13972            handle.spawn_nonce(&none_spec.module_id).is_none(),
13973            "no nonce record for a process that will never present one"
13974        );
13975
13976        // Control: the subc-wire path is unchanged by the branch above.
13977        let wire_spec = spec(Vec::new());
13978        let mut wire = Command::new("/nonexistent");
13979        let wire_handoff =
13980            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
13981                .expect("subc-wire spawn args apply");
13982        let wire_fd_env = wire
13983            .as_std()
13984            .get_envs()
13985            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
13986            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
13987        #[cfg(unix)]
13988        assert_eq!(
13989            wire_fd_env,
13990            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
13991            "a subc-wire spawn names the pipe it will receive at descriptor 3"
13992        );
13993        #[cfg(windows)]
13994        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
13995        #[cfg(not(any(unix, windows)))]
13996        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
13997        let wire_args: Vec<String> = wire
13998            .as_std()
13999            .get_args()
14000            .map(|a| a.to_string_lossy().into_owned())
14001            .collect();
14002        assert_eq!(
14003            wire_args,
14004            vec![
14005                SUBC_ARG.to_string(),
14006                connection_file.to_string_lossy().into_owned()
14007            ],
14008            "a subc-wire spawn still carries --subc <path>"
14009        );
14010        assert_eq!(
14011            wire.as_std()
14012                .get_envs()
14013                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
14014            !cfg!(unix),
14015            "only Windows supplies the environment nonce"
14016        );
14017        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
14018    }
14019
14020    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
14021    /// spec tries to set it; only a swap candidate carries it.
14022    ///
14023    /// "Set it only on candidates" is not enough, because spawn applies the
14024    /// spec's env verbatim and the daemon's own environment is inherited: either
14025    /// could hand a plain restart the swap role, and a module reading it would
14026    /// warm on its long swap budget while callers wait. Asserted as an explicit
14027    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
14028    /// test above gives.
14029    #[test]
14030    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
14031        let role = |command: &Command| {
14032            command
14033                .as_std()
14034                .get_envs()
14035                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
14036                .last()
14037                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
14038        };
14039        let forged = spec(vec![(
14040            SUBC_SPAWN_ROLE_ENV.to_string(),
14041            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
14042        )]);
14043
14044        let mut plain = Command::new("/nonexistent");
14045        apply_child_env(&mut plain, &forged);
14046        apply_spawn_role(&mut plain, SpawnRole::Plain);
14047        assert_eq!(
14048            role(&plain),
14049            Some(None),
14050            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
14051        );
14052
14053        let mut candidate = Command::new("/nonexistent");
14054        apply_child_env(&mut candidate, &spec(Vec::new()));
14055        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
14056        assert_eq!(
14057            role(&candidate),
14058            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
14059        );
14060    }
14061
14062    /// Daemon-private capture retention keys never reach the child.
14063    ///
14064    /// cortexkit-log exposes retention as a Rust struct with no environment
14065    /// names, so these entries are supervisor metadata. Passing them through
14066    /// would invent a public child-process contract by accident.
14067    #[test]
14068    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
14069        let mut command = Command::new("/nonexistent");
14070        apply_child_env(
14071            &mut command,
14072            &spec(vec![
14073                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
14074                ("KEPT".to_string(), "yes".to_string()),
14075            ]),
14076        );
14077        let keys: Vec<String> = command
14078            .as_std()
14079            .get_envs()
14080            .filter(|(_, value)| value.is_some())
14081            .map(|(key, _)| key.to_string_lossy().into_owned())
14082            .collect();
14083        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
14084        assert!(
14085            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
14086            "daemon-private capture key leaked to the child: {keys:?}"
14087        );
14088    }
14089}
14090
14091#[cfg(test)]
14092mod jitter_tests {
14093    use super::jittered_health_delay;
14094    use std::{collections::HashSet, time::Duration};
14095
14096    /// Module ids drawn from a real fleet, so the dispersal claim is about names
14097    /// that actually occur rather than invented ones.
14098    ///
14099    /// This is a SAMPLE, not a registry: the property under test is that distinct
14100    /// ids disperse, which holds for any set of distinct strings. Several entries
14101    /// are already historical (modules get renamed), and that costs nothing here --
14102    /// but it means a reader must not mistake this for the live module set, and a
14103    /// rename sweep will match it without there being anything to change.
14104    const FLEET: [&str; 14] = [
14105        "aft",
14106        "alfonso-core",
14107        "magic-context",
14108        "broca",
14109        "thalamus",
14110        "quota",
14111        "engram",
14112        "plexus",
14113        "cerebellum",
14114        "astrocyte",
14115        "synapse",
14116        "subc-mcp",
14117        "cortexkit-credentials",
14118        "subc-federation",
14119    ];
14120
14121    /// Probes must not converge after a fleet-wide restart.
14122    ///
14123    /// This is the property the jitter exists for: every module reconnects at
14124    /// once, and without dispersal all fourteen would then probe on the same
14125    /// tick forever. Nothing failed visibly when this went untested -- a
14126    /// convergent fleet still probes correctly, just in a burst, so the symptom
14127    /// is a periodic load spike that looks like whatever else is running.
14128    #[test]
14129    fn probe_delays_disperse_across_the_fleet() {
14130        let cadence = Duration::from_secs(30);
14131        let delays: HashSet<Duration> = FLEET
14132            .iter()
14133            .map(|id| jittered_health_delay(id, 0, cadence))
14134            .collect();
14135        assert_eq!(
14136            delays.len(),
14137            FLEET.len(),
14138            "every supervised module must land on its own probe offset"
14139        );
14140    }
14141
14142    /// The offset may only ever DELAY a probe, never bring it forward.
14143    ///
14144    /// A delay below the cadence would probe a module more often than
14145    /// configured, which is the opposite of what an operator asked for and
14146    /// would tighten the failure budget without anyone changing it.
14147    #[test]
14148    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
14149        let cadence = Duration::from_secs(30);
14150        let span = cadence / 10;
14151        for id in FLEET {
14152            for probe_index in 0..8 {
14153                let delay = jittered_health_delay(id, probe_index, cadence);
14154                assert!(
14155                    delay >= cadence,
14156                    "{id}#{probe_index}: jitter must not shorten the cadence"
14157                );
14158                assert!(
14159                    delay < cadence + span,
14160                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
14161                );
14162            }
14163        }
14164    }
14165
14166    /// A module keeps its offset across daemon restarts.
14167    ///
14168    /// The delay is derived rather than randomised precisely so a restart does
14169    /// not re-roll every module into a fresh chance of collision. A random
14170    /// source would satisfy the dispersal test above and quietly lose this.
14171    #[test]
14172    fn a_module_offset_is_stable_across_restarts() {
14173        let cadence = Duration::from_secs(30);
14174        for id in FLEET {
14175            assert_eq!(
14176                jittered_health_delay(id, 0, cadence),
14177                jittered_health_delay(id, 0, cadence),
14178                "{id}: the same module and probe index must produce the same offset"
14179            );
14180        }
14181    }
14182
14183    /// A zero cadence disables probing rather than producing a busy loop.
14184    #[test]
14185    fn zero_cadence_yields_zero_delay() {
14186        assert_eq!(
14187            jittered_health_delay("aft", 0, Duration::ZERO),
14188            Duration::ZERO
14189        );
14190    }
14191}
14192
14193#[cfg(all(test, target_os = "linux"))]
14194mod cgroup_placement_tests {
14195    use super::{
14196        apply_cgroup_placement, remove_module_cgroup, ModuleProtocol, ModuleSpec, SuperviseError,
14197        SupervisedChild,
14198    };
14199    use crate::stderr_tail::{StderrRing, StderrTailConfig};
14200    use cortexkit_test_support::ScratchDir;
14201    use std::{
14202        fs, io,
14203        path::{Path, PathBuf},
14204        sync::{Arc, Mutex},
14205    };
14206    use tokio::process::Command;
14207
14208    #[tokio::test]
14209    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
14210        use super::*;
14211        let dir = ScratchDir::new("unique-spawn-cgroups");
14212        let root = PathBuf::from(format!(
14213            "/sys/fs/cgroup/subc-unique-test-{}-{}",
14214            std::process::id(),
14215            unix_ms_now()
14216        ));
14217        if let Err(error) = fs::create_dir(&root) {
14218            assert!(
14219                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
14220                "required cgroup test cannot execute: {error}"
14221            );
14222            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
14223            return;
14224        }
14225        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
14226        let group_count = || {
14227            fs::read_dir(root.join("subc-modules"))
14228                .unwrap()
14229                .map(|entry| entry.unwrap().file_type().unwrap())
14230                .filter(|kind| kind.is_dir())
14231                .count()
14232        };
14233        let supervisor =
14234            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
14235                .with_cgroup_placement(Some(placement.clone()));
14236        let runtime = supervisor.runtime_config();
14237        let mut spec = ModuleSpec {
14238            module_id: "unique-spawn".into(),
14239            program: PathBuf::from("/bin/sleep"),
14240            args: vec!["60".into()],
14241            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14242                .into_iter()
14243                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
14244                .collect(),
14245            reserved: false,
14246            reserved_prefixes: vec![],
14247            protocol: ModuleProtocol::None,
14248            overlap: Default::default(),
14249        };
14250        let spawn = |spec: &ModuleSpec| {
14251            spawn_child(
14252                spec,
14253                None,
14254                None,
14255                &runtime.stderr_ring,
14256                None,
14257                &runtime.child_roster,
14258                Some(&placement),
14259            )
14260            .unwrap()
14261        };
14262        let mut live = spawn(&spec);
14263        for _ in 0..3 {
14264            // A new process can enter the old slot while retirement is pending.
14265            let next = spawn(&spec);
14266            assert_ne!(live.module_id, next.module_id);
14267            live.start_kill().unwrap();
14268            live.wait().await.unwrap();
14269            live = next;
14270            assert!(
14271                live.child.try_wait().unwrap().is_none(),
14272                "retiring the old slot must not kill the replacement"
14273            );
14274            assert_eq!(
14275                group_count(),
14276                1,
14277                "only the live spawn's cgroup should remain"
14278            );
14279        }
14280        supervisor.begin_daemon_shutdown();
14281        let reap = tokio::spawn(async move {
14282            live.wait().await.unwrap();
14283        });
14284        supervisor
14285            .end_children_for_daemon_shutdown(false, std::future::pending())
14286            .await;
14287        reap.await.unwrap();
14288        assert_eq!(group_count(), 0);
14289        // A normal exit uses the same tree-cleanup path as a killed spawn.
14290        spec.program = PathBuf::from("/bin/true");
14291        spec.args.clear();
14292        let fresh_roster = ChildRoster::default();
14293        let mut short = spawn_child(
14294            &spec,
14295            None,
14296            None,
14297            &runtime.stderr_ring,
14298            None,
14299            &fresh_roster,
14300            Some(&placement),
14301        )
14302        .unwrap();
14303        short.wait().await.unwrap();
14304        assert_eq!(group_count(), 0);
14305        spec.module_id = "_".repeat(255);
14306        let mut long_id = spawn_child(
14307            &spec,
14308            None,
14309            None,
14310            &runtime.stderr_ring,
14311            None,
14312            &fresh_roster,
14313            Some(&placement),
14314        )
14315        .unwrap();
14316        long_id.wait().await.unwrap();
14317        assert_eq!(
14318            group_count(),
14319            0,
14320            "valid long module IDs must not exceed cgroup NAME_MAX"
14321        );
14322        fs::remove_dir(root.join("subc-modules")).unwrap();
14323        fs::remove_dir(root).unwrap();
14324        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
14325    }
14326
14327    #[test]
14328    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
14329        let path = Path::new("/definitely-missing-subc-cgroup");
14330        let mut command = Command::new("true");
14331        let error = apply_cgroup_placement(
14332            &mut command,
14333            &ModuleSpec {
14334                module_id: "broken-cgroup".to_string(),
14335                program: PathBuf::from("true"),
14336                args: Vec::new(),
14337                env: Vec::new(),
14338                reserved: false,
14339                reserved_prefixes: Vec::new(),
14340                protocol: ModuleProtocol::Subc,
14341                overlap: Default::default(),
14342            },
14343            path,
14344        )
14345        .expect_err("a parent cgroup open failure must reject the supervised spawn");
14346        let reason = error.to_string();
14347
14348        assert!(
14349            matches!(error, SuperviseError::Cgroup { .. }),
14350            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
14351        );
14352        assert!(
14353            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
14354            "parent cgroup open failure must name cgroup.procs: {reason}"
14355        );
14356    }
14357
14358    #[tokio::test]
14359    async fn reaping_a_child_removes_its_empty_module_cgroup() {
14360        let root = ScratchDir::new("supervisor-reap-cgroup");
14361        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
14362        let placement = subc_cgroup::prepare_at(&root)
14363            .expect("prepare scratch cgroup root")
14364            .expect("scratch root has a cgroup.procs marker");
14365        let module_id = "reaped-module";
14366        let module = placement
14367            .module_path(module_id)
14368            .expect("create scratch module cgroup");
14369        let child = Command::new("true")
14370            .env("XDG_DATA_HOME", root.path())
14371            .env("XDG_RUNTIME_DIR", root.path())
14372            .env("XDG_CONFIG_HOME", root.path())
14373            .spawn()
14374            .expect("spawn short-lived child");
14375        let pid = child.id().expect("spawned child has pid");
14376        let mut child = SupervisedChild {
14377            child,
14378            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
14379            protocol: ModuleProtocol::Subc,
14380            module_id: module_id.to_string(),
14381            cgroup_placement: Some(placement),
14382            stdout_pump: None,
14383            stderr_pump: None,
14384            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
14385            spawned_at_ms: 0,
14386            spawned_from: PathBuf::from("true"),
14387            spawned_file_identity: None,
14388            process_start_time: None,
14389            process_identity: None,
14390            pid,
14391            roster_guard: None,
14392            #[cfg(target_os = "macos")]
14393            privacy_exec: None,
14394            spawn_failure: None,
14395        };
14396
14397        child.wait().await.expect("reap short-lived child");
14398
14399        assert!(
14400            !module.exists(),
14401            "reaping the supervised child must remove its empty cgroup"
14402        );
14403    }
14404
14405    #[test]
14406    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
14407        let root = ScratchDir::new("supervisor-non-empty-cgroup");
14408        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
14409        let placement = subc_cgroup::prepare_at(&root)
14410            .expect("prepare scratch cgroup root")
14411            .expect("scratch root has a cgroup.procs marker");
14412        let module = placement
14413            .module_path("surviving-module")
14414            .expect("create scratch module cgroup");
14415        fs::write(module.join("surviving-process"), b"still present")
14416            .expect("make scratch cgroup non-empty");
14417        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
14418
14419        remove_module_cgroup(&placement, "surviving-module");
14420
14421        let logs = crate::router::test_log::captured_logs(&logs);
14422        assert!(
14423            module.exists(),
14424            "failed removal must leave the cgroup intact"
14425        );
14426        assert!(
14427            logs.contains("could not remove module cgroup after process exit; continuing teardown")
14428                && logs.contains("surviving-module"),
14429            "best-effort removal must report the failure without returning it: {logs}"
14430        );
14431    }
14432
14433    #[test]
14434    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
14435        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
14436        let reason = SuperviseError::Spawn {
14437            program: PathBuf::from("/bin/true"),
14438            source: io::Error::from_raw_os_error(13),
14439            cgroup_path: Some(cgroup_path.clone()),
14440        }
14441        .to_string();
14442
14443        assert!(
14444            reason.contains(&cgroup_path.display().to_string()),
14445            "a pre_exec spawn failure must name the cgroup path: {reason}"
14446        );
14447    }
14448}
14449
14450#[cfg(test)]
14451mod spawn_subscriber_lag_tests {
14452    use super::*;
14453
14454    /// A subscriber whose connection stops draining is dropped once its frame
14455    /// channel fills. The client must learn that from a terminal Error frame
14456    /// after the frames already queued for it, not from a stream that simply
14457    /// goes quiet.
14458    #[tokio::test]
14459    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
14460        let feed = SpawnEventFeed::default();
14461        feed.configure_incarnation("lag-incarnation".to_string());
14462        // A one-slot connection queue that nobody reads until the emits are
14463        // done: the forwarder parks on it and the subscriber channel fills.
14464        let (tx, mut rx) = mpsc::channel(1);
14465        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
14466            .expect("subscribe");
14467        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
14468        for index in 0..emitted {
14469            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
14470            // Let the forwarder take what it can so the fill point is the
14471            // subscriber channel, not a scheduling accident.
14472            tokio::task::yield_now().await;
14473        }
14474        assert_eq!(
14475            feed.subscriber_count(),
14476            0,
14477            "the lagged subscriber must be removed"
14478        );
14479
14480        let mut data = Vec::new();
14481        let mut last = None;
14482        loop {
14483            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14484                .await
14485                .expect("the forwarder must finish once the subscriber is dropped");
14486            let Some(outbound) = next else { break };
14487            let frame = outbound.frame;
14488            if frame.header.ty == FrameType::StreamData {
14489                assert!(last.is_none(), "no data may follow the terminal frame");
14490                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
14491                data.push(event.cursor.seq);
14492            } else {
14493                assert!(last.is_none(), "exactly one terminal frame");
14494                last = Some(frame);
14495            }
14496        }
14497        assert!(!data.is_empty(), "queued frames drain before the terminal");
14498        for pair in data.windows(2) {
14499            assert_eq!(
14500                pair[1],
14501                pair[0] + 1,
14502                "queued frames arrive dense and in order"
14503            );
14504        }
14505        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
14506        assert_eq!(terminal.header.ty, FrameType::Error);
14507        assert_eq!(terminal.header.corr, 7);
14508        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
14509        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
14510        let detail = body.detail.expect("lagged error carries detail");
14511        assert_eq!(
14512            detail["first_undelivered_cursor"]["seq"],
14513            data.last().unwrap() + 1,
14514            "the named cursor is the first event the subscriber did not receive"
14515        );
14516        assert_eq!(
14517            detail["first_undelivered_cursor"]["daemon_incarnation"],
14518            "lag-incarnation"
14519        );
14520    }
14521}
14522
14523#[cfg(test)]
14524mod terminal_history_read_concurrency_tests {
14525    use super::*;
14526    use crate::terminal_journal::read_pause;
14527    use cortexkit_test_support::ScratchDir;
14528    use std::sync::mpsc as std_mpsc;
14529
14530    fn journaled_ring(
14531        journal: &Arc<crate::terminal_journal::TerminalJournal>,
14532    ) -> Arc<Mutex<TerminalRing>> {
14533        Arc::new(Mutex::new(
14534            TerminalRing::new(TerminalRingConfig::default(), 1)
14535                .with_journal(Some(Arc::clone(journal))),
14536        ))
14537    }
14538
14539    fn crash(at_ms: u64) -> ExitReport {
14540        ExitReport {
14541            kind: ExitKind::Crash,
14542            code: Some(1),
14543            signal: None,
14544            at_ms,
14545        }
14546    }
14547
14548    /// Record an exit on another thread and report whether it finished within
14549    /// `bound`. The recorder thread is left running if it did not.
14550    fn record_within(
14551        module_id: &'static str,
14552        ring: &Arc<Mutex<TerminalRing>>,
14553        at_ms: u64,
14554        bound: Duration,
14555    ) -> bool {
14556        let ring = Arc::clone(ring);
14557        let (done, done_rx) = std_mpsc::channel();
14558        std::thread::spawn(move || {
14559            record_terminal(
14560                module_id,
14561                &ring,
14562                &SpawnEventFeed::default(),
14563                &crash(at_ms),
14564                TerminalDisposition::Restarting,
14565            );
14566            let _ = done.send(());
14567        });
14568        done_rx.recv_timeout(bound).is_ok()
14569    }
14570
14571    /// A history read in progress must not hold the journal writer (which every
14572    /// module's exit recording needs) or the module's own ring. Exits recorded
14573    /// while the read is paused complete promptly; the paused read answers as of
14574    /// the moment it started, and the next read has each exit exactly once.
14575    #[test]
14576    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
14577        let dir = ScratchDir::new("terminal-history-concurrent-read");
14578        let path = dir.join("terminals.jsonl");
14579        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
14580            path.clone(),
14581            "daemon".into(),
14582        ));
14583        let reader_ring = journaled_ring(&journal);
14584        let other_ring = journaled_ring(&journal);
14585        assert!(record_within(
14586            "reader-module",
14587            &reader_ring,
14588            10,
14589            Duration::from_secs(5)
14590        ));
14591
14592        let (started, release) = read_pause::install(&path);
14593        let reading = {
14594            let ring = Arc::clone(&reader_ring);
14595            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
14596        };
14597        started
14598            .recv_timeout(Duration::from_secs(5))
14599            .expect("the history read reached its pause");
14600
14601        let bound = Duration::from_secs(1);
14602        assert!(
14603            record_within("other-module", &other_ring, 20, bound),
14604            "another module's exit waited on a history read (journal writer held)"
14605        );
14606        assert!(
14607            record_within("reader-module", &reader_ring, 30, bound),
14608            "the read module's own exit waited on its history read (ring held)"
14609        );
14610
14611        drop(release);
14612        let paused = reading.join().unwrap();
14613        assert_eq!(
14614            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
14615            vec![10],
14616            "an exit recorded after the read began lands in neither half of it"
14617        );
14618        assert_eq!(paused.journal_skipped_lines, 0);
14619        assert_eq!(paused.journal_read_errors, 0);
14620
14621        let after = durable_terminal_history_of(&reader_ring, "reader-module");
14622        assert_eq!(
14623            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
14624            vec![10, 30],
14625            "the next read merges ring and journal with no duplicate"
14626        );
14627        assert_eq!(after.journal_skipped_lines, 0);
14628    }
14629}
14630
14631/// What a restart does with the exited process's output readers. These drive
14632/// the same settle functions the supervisor calls, with a reader the test
14633/// holds, so a reader that has not been scheduled by the bound is a controlled
14634/// input rather than something only a loaded machine produces.
14635#[cfg(test)]
14636mod stderr_settle_tests {
14637    use std::{
14638        future::Future,
14639        io,
14640        pin::Pin,
14641        sync::{Arc, Mutex},
14642        task::{Context, Poll},
14643        time::Duration,
14644    };
14645
14646    use tokio::{
14647        io::{AsyncRead, ReadBuf},
14648        sync::oneshot,
14649        time::Instant,
14650    };
14651
14652    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
14653    use crate::stderr_tail::{
14654        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
14655        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
14656    };
14657
14658    const BOUND: Duration = Duration::from_millis(250);
14659
14660    /// Yields `before`, then stays pending until the gate is released, then
14661    /// yields `after` and reaches EOF. The bytes after the gate were written
14662    /// by a process that has already exited; only the reader is behind.
14663    struct HeldReader {
14664        before: Option<Vec<u8>>,
14665        gate: Option<oneshot::Receiver<()>>,
14666        after: io::Cursor<Vec<u8>>,
14667    }
14668
14669    impl AsyncRead for HeldReader {
14670        fn poll_read(
14671            mut self: Pin<&mut Self>,
14672            cx: &mut Context<'_>,
14673            buf: &mut ReadBuf<'_>,
14674        ) -> Poll<io::Result<()>> {
14675            if let Some(bytes) = self.before.take() {
14676                buf.put_slice(&bytes);
14677                return Poll::Ready(Ok(()));
14678            }
14679            if let Some(gate) = self.gate.as_mut() {
14680                match Pin::new(gate).poll(cx) {
14681                    Poll::Pending => return Poll::Pending,
14682                    Poll::Ready(_) => self.gate = None,
14683                }
14684            }
14685            Pin::new(&mut self.after).poll_read(cx, buf)
14686        }
14687    }
14688
14689    struct DiscardSink;
14690
14691    impl OutputSink for DiscardSink {
14692        fn write_line(&mut self, _line: &[u8]) {}
14693    }
14694
14695    fn line(text: &str) -> TailEntry {
14696        TailEntry::Line {
14697            text: text.to_string(),
14698            truncated: false,
14699            at_ms: None,
14700        }
14701    }
14702
14703    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
14704        ring.lock().unwrap()
14705    }
14706
14707    /// Start a reader for a new process generation that delivers `before`
14708    /// immediately and `after` only once the returned sender fires (or is
14709    /// dropped).
14710    fn held_pump(
14711        ring: &Arc<Mutex<StderrRing>>,
14712        before: &str,
14713        after: &str,
14714    ) -> (StderrPump, oneshot::Sender<()>) {
14715        let generation = lock(ring).begin_process();
14716        let (release, gate) = oneshot::channel();
14717        let reader = HeldReader {
14718            before: Some(before.as_bytes().to_vec()),
14719            gate: Some(gate),
14720            after: io::Cursor::new(after.as_bytes().to_vec()),
14721        };
14722        let task = tokio::spawn(pump_stderr_to(
14723            reader,
14724            Arc::clone(ring),
14725            generation,
14726            DiscardSink,
14727        ));
14728        (StderrPump { task, generation }, release)
14729    }
14730
14731    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
14732        for _ in 0..1000 {
14733            if done(&lock(ring)) {
14734                return;
14735            }
14736            tokio::time::sleep(Duration::from_millis(1)).await;
14737        }
14738        panic!(
14739            "ring never reached the expected state: {:?}",
14740            lock(ring).snapshot(None, None)
14741        );
14742    }
14743
14744    #[tokio::test(start_paused = true)]
14745    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
14746        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
14747        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
14748
14749        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
14750        let before_release = lock(&ring).snapshot(None, None);
14751        assert!(
14752            matches!(before_release.capture, CaptureState::Incomplete { .. }),
14753            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
14754        );
14755
14756        // The restart: the next process starts and writes before the old
14757        // reader catches up.
14758        let next = lock(&ring).begin_process();
14759        lock(&ring).push_line_from(next, "next process booting");
14760        release.send(()).unwrap();
14761        wait_until(&ring, |ring| {
14762            ring.snapshot(None, None).capture == CaptureState::Captured
14763        })
14764        .await;
14765
14766        assert_eq!(
14767            untimed(lock(&ring).snapshot(None, None).entries),
14768            vec![
14769                line("booting"),
14770                line("config error: missing storage"),
14771                TailEntry::ProcessStart,
14772                line("next process booting"),
14773            ],
14774            "the crash's last line must survive a slow reader and stay in the crashed process's section"
14775        );
14776    }
14777
14778    #[tokio::test(start_paused = true)]
14779    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
14780    ) {
14781        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
14782        // `_held` is never fired: a descendant keeps the pipe open for the
14783        // whole test.
14784        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
14785
14786        let started = Instant::now();
14787        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
14788        assert_eq!(
14789            started.elapsed(),
14790            BOUND,
14791            "the restart must wait exactly the bound for a pipe that stays open, no longer"
14792        );
14793
14794        let next = lock(&ring).begin_process();
14795        lock(&ring).push_line_from(next, "next process booting");
14796        tokio::time::sleep(Duration::from_secs(60)).await;
14797
14798        let snapshot = lock(&ring).snapshot(None, None);
14799        match &snapshot.capture {
14800            CaptureState::Incomplete { reason } => assert!(
14801                reason.contains("had not reached EOF") && reason.contains("250ms"),
14802                "the reason must say what is missing and after how long: {reason}"
14803            ),
14804            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
14805        }
14806        assert_eq!(
14807            untimed(snapshot.entries),
14808            vec![
14809                line("parent exiting"),
14810                TailEntry::ProcessStart,
14811                line("next process booting"),
14812            ]
14813        );
14814    }
14815
14816    #[tokio::test(start_paused = true)]
14817    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
14818        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
14819        let (pump, release) = held_pump(&ring, "one\n", "two\n");
14820        release.send(()).unwrap();
14821
14822        settle_stderr_pump("clean", &ring, pump, BOUND).await;
14823
14824        let snapshot = lock(&ring).snapshot(None, None);
14825        assert_eq!(snapshot.capture, CaptureState::Captured);
14826        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
14827    }
14828
14829    #[tokio::test(start_paused = true)]
14830    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
14831        const BURST: usize = 150;
14832        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
14833        let path = capture.join("two-pipe.stderr.log");
14834        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
14835        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
14836        let generation = lock(&ring).begin_process();
14837        let stdout = (0..BURST)
14838            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
14839            .collect::<String>();
14840        let stderr = (0..BURST)
14841            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
14842            .collect::<String>();
14843        // Model a child that has written both bursts and exited. Hold the stdout
14844        // reader after one line and a partial prefix, as if pipe readiness or
14845        // capture-file writes had delayed it. The final line also needs EOF to
14846        // flush: neither complete nor partial buffered output may be discarded.
14847        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
14848        let (release, gate) = oneshot::channel();
14849        let reader = HeldReader {
14850            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
14851            gate: Some(gate),
14852            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
14853        };
14854        let (finished, completion) = oneshot::channel();
14855        let stdout_sink = sink.clone();
14856        let pump = tokio::spawn(async move {
14857            pump_stdout_to(reader, stdout_sink).await;
14858            finished.send(()).unwrap();
14859        });
14860        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
14861
14862        let started = Instant::now();
14863        settle_stdout_pump("two-pipe", pump, BOUND).await;
14864        assert_eq!(
14865            started.elapsed(),
14866            BOUND,
14867            "a late reader must not hold up restart"
14868        );
14869        assert_eq!(
14870            std::fs::read_to_string(&path).unwrap().lines().count(),
14871            BURST + 1
14872        );
14873
14874        release
14875            .send(())
14876            .expect("stdout reader must survive the exit drain bound");
14877        tokio::time::timeout(Duration::from_secs(1), completion)
14878            .await
14879            .expect("released stdout reader must reach EOF")
14880            .expect("stdout capture must finish at EOF, not be cancelled");
14881        let contents = std::fs::read_to_string(&path).unwrap();
14882        let mut lines = contents
14883            .lines()
14884            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
14885            .collect::<Vec<_>>();
14886        let mut expected = (0..BURST)
14887            .flat_map(|i| {
14888                [
14889                    format!("out-{i:04}-{}", "o".repeat(64)),
14890                    format!("err-{i:04}-{}", "e".repeat(64)),
14891                ]
14892            })
14893            .collect::<Vec<_>>();
14894        lines.sort_unstable();
14895        expected.sort_unstable();
14896        assert_eq!(
14897            lines, expected,
14898            "both pipes must retain every complete line exactly once"
14899        );
14900    }
14901}
14902
14903/// Containment of a module's process tree (issue #109).
14904///
14905/// The behaviour these defend against is a module helper surviving its module:
14906/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
14907/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
14908/// compounds it.
14909///
14910/// They run against the SUPERVISOR rather than the job-object crate because the
14911/// claim is about teardown: a crate-level test proves a job can reap a tree, not
14912/// that the daemon's drain path reaches it.
14913///
14914/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
14915/// lane there is a separate containment path with its own tests.
14916#[cfg(all(test, windows))]
14917mod job_containment_tests {
14918    use super::*;
14919    use cortexkit_test_support::ScratchDir;
14920    use std::{
14921        path::{Path, PathBuf},
14922        sync::{Arc, Mutex},
14923        time::{Duration, Instant},
14924    };
14925
14926    /// The stub, expected beside this test executable.
14927    ///
14928    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
14929    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
14930    /// failure then reads as a broken test rather than an unbuilt dependency.
14931    fn stub_path() -> PathBuf {
14932        let mut path = std::env::current_exe().expect("current_exe available in tests");
14933        path.pop();
14934        path.pop();
14935        path.push("fake-aft-stub.exe");
14936        assert!(
14937            path.exists(),
14938            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
14939             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
14940            path.display()
14941        );
14942        path
14943    }
14944
14945    /// Poll for the grandchild pid the stub records, and parse it.
14946    fn read_grandchild_pid(path: &Path) -> u32 {
14947        let deadline = Instant::now() + Duration::from_secs(10);
14948        loop {
14949            if let Ok(contents) = std::fs::read_to_string(path) {
14950                if let Ok(pid) = contents.trim().parse() {
14951                    return pid;
14952                }
14953            }
14954            assert!(
14955                Instant::now() < deadline,
14956                "the stub never recorded a grandchild pid at {}",
14957                path.display()
14958            );
14959            std::thread::sleep(Duration::from_millis(10));
14960        }
14961    }
14962
14963    /// Everything one fixture run needs, so the two tests below differ in exactly
14964    /// one place: whether the child is contained.
14965    struct Fixture {
14966        _dir: ScratchDir,
14967        module_id: String,
14968        grandchild: u32,
14969        child: Option<SupervisedChild>,
14970        registry: Arc<Registry>,
14971        snapshot: Arc<Mutex<SupervisorSnapshot>>,
14972        terminal_ring: Arc<Mutex<TerminalRing>>,
14973        spawn_events: SpawnEventFeed,
14974    }
14975
14976    fn fixture(label: &str, module_id: &str) -> Fixture {
14977        let dir = ScratchDir::new(label);
14978        let pid_file = dir.join("grandchild.pid");
14979        let supervisor = Supervisor::new_for_test(
14980            Arc::new(Registry::default()),
14981            RestartPolicy::new(3, Duration::ZERO),
14982        );
14983        let runtime = supervisor.runtime_config();
14984        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14985        let spec = ModuleSpec {
14986            module_id: module_id.to_string(),
14987            program: stub_path(),
14988            // Zero args deliberately: a `--subc` argument would make the stub dial
14989            // a daemon that is not there, and the failure would land in the same
14990            // stderr ring this fixture exists to keep quiet.
14991            args: Vec::new(),
14992            env: vec![
14993                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
14994                (
14995                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
14996                    pid_file.display().to_string(),
14997                ),
14998            ],
14999            reserved: false,
15000            reserved_prefixes: Vec::new(),
15001            protocol: ModuleProtocol::Subc,
15002            overlap: Default::default(),
15003        };
15004        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
15005            .expect("spawn the supervised fixture");
15006        let grandchild = read_grandchild_pid(&pid_file);
15007        Fixture {
15008            _dir: dir,
15009            module_id: module_id.to_string(),
15010            grandchild,
15011            child: Some(child),
15012            registry: Arc::new(Registry::default()),
15013            snapshot,
15014            terminal_ring: Arc::clone(&runtime.terminal_ring),
15015            spawn_events: SpawnEventFeed::default(),
15016        }
15017    }
15018
15019    impl Fixture {
15020        /// Drain through the supervisor's own teardown path.
15021        async fn drain(&mut self) {
15022            let child = self
15023                .child
15024                .take()
15025                .expect("the fixture child is still present");
15026            drain_child_to_state(
15027                &self.module_id,
15028                ModuleProtocol::Subc,
15029                // No forwarding table in this fixture, so nothing reaches the
15030                // child over a connection.
15031                StopNotice::NotSent,
15032                &self.registry,
15033                None,
15034                &self.snapshot,
15035                &self.terminal_ring,
15036                &self.spawn_events,
15037                child,
15038                Duration::from_millis(500),
15039                ModuleState::Stopped,
15040                Some(false),
15041            )
15042            .await
15043            .expect("drain the supervised fixture");
15044        }
15045    }
15046
15047    /// Teardown reaps the grandchild, not merely the direct child.
15048    ///
15049    /// This is the assertion the change exists for. Before containment the
15050    /// grandchild survived: it is a separate process, and `start_kill` is
15051    /// `TerminateProcess` scoped to one pid.
15052    #[tokio::test]
15053    async fn teardown_reaps_the_grandchild() {
15054        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
15055        let grandchild = fixture.grandchild;
15056
15057        assert!(
15058            subc_jobobject::process_exists(grandchild),
15059            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
15060        );
15061
15062        fixture.drain().await;
15063
15064        assert!(
15065            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
15066            "grandchild {grandchild} outlived module teardown: the tree was not contained"
15067        );
15068    }
15069
15070    /// The mutation control: with containment withheld, the grandchild survives
15071    /// the same kill.
15072    ///
15073    /// This is the defect reproduction from #109 — a direct-child kill reaches
15074    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
15075    /// supervisor because `spawn_and_mark_running` now always contains on
15076    /// Windows, which is the point: there is no longer a path that spawns
15077    /// uncontained, so the control has to construct one.
15078    ///
15079    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
15080    /// grandchild ever dies here, that test is passing for a reason unrelated to
15081    /// the job object and the containment claim is unproven.
15082    #[test]
15083    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
15084        let dir = ScratchDir::new("teardown-uncontained");
15085        let pid_file = dir.join("grandchild.pid");
15086        let mut child = std::process::Command::new(stub_path())
15087            .env("FAKE_AFT_NEVER_CONNECT", "1")
15088            .env(
15089                "FAKE_AFT_GRANDCHILD_PID_FILE",
15090                pid_file.display().to_string(),
15091            )
15092            .stdin(std::process::Stdio::null())
15093            .stdout(std::process::Stdio::null())
15094            .stderr(std::process::Stdio::null())
15095            .spawn()
15096            .expect("spawn the uncontained fixture");
15097        let grandchild = read_grandchild_pid(&pid_file);
15098
15099        // Exactly what the pre-fix teardown did: kill the direct child.
15100        child.kill().expect("kill the direct child");
15101        let _ = child.wait();
15102
15103        assert!(
15104            subc_jobobject::process_exists(grandchild),
15105            "grandchild {grandchild} died with the direct child, so this control no longer \
15106             distinguishes contained from uncontained teardown and the regression test is \
15107             passing vacuously"
15108        );
15109
15110        // The orphan this control demonstrates is the leak the fix prevents, so
15111        // the control must not leave one behind.
15112        kill_tree(grandchild);
15113    }
15114
15115    /// Crash durability: closing the containment handle reaps the tree with no
15116    /// teardown code running at all.
15117    ///
15118    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
15119    /// call anything — and it is why containment is a kernel property of the
15120    /// handle rather than a step in the drain. Discovered by getting the
15121    /// mutation control wrong: clearing `job` to "disable" containment instead
15122    /// killed the tree, which is the guarantee, not a mistake.
15123    #[tokio::test]
15124    async fn dropping_containment_reaps_the_grandchild() {
15125        let mut fixture = fixture("drop-containment", "tree-drop");
15126        let grandchild = fixture.grandchild;
15127
15128        assert!(subc_jobobject::process_exists(grandchild));
15129
15130        // No `drain` call, no kill: dropping the handle is the entire mechanism.
15131        fixture.child.as_mut().expect("child present").job = None;
15132
15133        assert!(
15134            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
15135            "grandchild {grandchild} survived the containment handle closing, so a daemon \
15136             crash would leave the tree behind"
15137        );
15138    }
15139
15140    /// Kill a pid and its tree, then confirm it is gone.
15141    fn kill_tree(pid: u32) {
15142        let _ = std::process::Command::new("taskkill.exe")
15143            .args(["/PID", &pid.to_string(), "/T", "/F"])
15144            .stdin(std::process::Stdio::null())
15145            .stdout(std::process::Stdio::null())
15146            .stderr(std::process::Stdio::null())
15147            .status();
15148        assert!(
15149            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
15150            "could not clean up grandchild {pid}"
15151        );
15152    }
15153}
15154
15155#[cfg(test)]
15156mod privacy_trampoline_configuration_tests {
15157    #[tokio::test]
15158    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15159    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
15160        #[cfg(target_os = "macos")]
15161        {
15162            let supervisor = super::Supervisor::new(
15163                std::sync::Arc::new(crate::Registry::default()),
15164                super::RestartPolicy::default(),
15165            );
15166            let error = supervisor.spawn(spec()).unwrap_err();
15167            assert!(
15168                error
15169                    .to_string()
15170                    .contains("no privacy trampoline configured"),
15171                "{error}"
15172            );
15173        }
15174    }
15175
15176    #[tokio::test]
15177    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15178    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
15179        #[cfg(target_os = "macos")]
15180        {
15181            let supervisor = super::Supervisor::new(
15182                std::sync::Arc::new(crate::Registry::default()),
15183                super::RestartPolicy::default(),
15184            )
15185            .with_privacy_trampoline(std::env::current_exe().unwrap());
15186            let error = supervisor.spawn(spec()).unwrap_err();
15187            assert!(
15188                error
15189                    .to_string()
15190                    .contains("binary does not implement the privacy trampoline protocol"),
15191                "{error}"
15192            );
15193        }
15194    }
15195
15196    #[cfg(target_os = "macos")]
15197    fn spec() -> super::ModuleSpec {
15198        super::ModuleSpec {
15199            module_id: "privacy-configuration".into(),
15200            program: "/bin/sleep".into(),
15201            args: vec!["30".into()],
15202            env: vec![],
15203            reserved: false,
15204            reserved_prefixes: vec![],
15205            protocol: subc_control::ModuleProtocol::None,
15206            overlap: super::ModuleOverlap::Exclusive,
15207        }
15208    }
15209}
15210
15211#[cfg(test)]
15212mod privacy_exec_boundary_tests {
15213    #[cfg(target_os = "macos")]
15214    use super::*;
15215    #[cfg(target_os = "macos")]
15216    use std::{
15217        io::{Read, Write},
15218        net::{TcpListener, TcpStream},
15219    };
15220
15221    /// Unit-test-only pause at the actual early image read, not at a later
15222    /// status read. Production supervisors never inspect this environment key.
15223    #[cfg(target_os = "macos")]
15224    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
15225        if let Some((_, path)) = spec
15226            .env
15227            .iter()
15228            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
15229        {
15230            let mut barrier = TcpStream::connect(path).unwrap();
15231            barrier
15232                .set_read_timeout(Some(Duration::from_secs(30)))
15233                .unwrap();
15234            barrier.write_all(&pid.to_ne_bytes()).unwrap();
15235            let mut release = [0];
15236            barrier.read_exact(&mut release).unwrap();
15237            assert_eq!(&release, b"X");
15238        }
15239    }
15240
15241    #[cfg(target_os = "macos")]
15242    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
15243        ModuleSpec {
15244            module_id: "privacy-boundary".into(),
15245            program: program.into(),
15246            args: args.iter().map(|arg| (*arg).into()).collect(),
15247            env: vec![],
15248            reserved: false,
15249            reserved_prefixes: vec![],
15250            protocol: ModuleProtocol::None,
15251            overlap: ModuleOverlap::Exclusive,
15252        }
15253    }
15254
15255    #[cfg(target_os = "macos")]
15256    async fn accept(listener: TcpListener) -> TcpStream {
15257        // Socket readiness, not elapsed time, establishes both pause points.
15258        let listener = tokio::net::TcpListener::from_std({
15259            listener.set_nonblocking(true).unwrap();
15260            listener
15261        })
15262        .unwrap();
15263        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
15264            .await
15265            .unwrap()
15266            .unwrap();
15267        let stream = stream.into_std().unwrap();
15268        stream.set_nonblocking(false).unwrap();
15269        stream
15270            .set_read_timeout(Some(Duration::from_secs(30)))
15271            .unwrap();
15272        stream
15273    }
15274
15275    #[tokio::test]
15276    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15277    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
15278        #[cfg(target_os = "macos")]
15279        {
15280            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
15281            // Loopback sockets also work when the replay adapter's TMPDIR is
15282            // longer than Darwin's Unix-domain socket path limit.
15283            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
15284            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
15285            let record = root.join("live-children.json");
15286            let supervisor =
15287                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15288                    .with_live_children_record(&record);
15289            let runtime = supervisor.runtime_config();
15290            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15291            let mut spec = spec("/bin/sleep", &["30"]);
15292            spec.env = vec![
15293                (
15294                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
15295                    exec_listener.local_addr().unwrap().to_string(),
15296                ),
15297                (
15298                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
15299                    sample_listener.local_addr().unwrap().to_string(),
15300                ),
15301            ];
15302            let spawn = tokio::task::spawn_blocking(move || {
15303                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
15304            });
15305            let mut sample = accept(sample_listener).await;
15306            let mut pid = [0; 4];
15307            sample.read_exact(&mut pid).unwrap();
15308            let pid = u32::from_ne_bytes(pid);
15309            let mut exec = accept(exec_listener).await;
15310            let mut ready = [0];
15311            exec.read_exact(&mut ready).unwrap();
15312            assert_eq!(&ready, b"R");
15313            // The early read is guaranteed to see a real, non-null trampoline
15314            // image: the fixture has reached its barrier and cannot exec yet.
15315            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
15316            assert_eq!(
15317                observe_spawned_image(pid).unwrap().executable,
15318                Some(trampoline)
15319            );
15320            sample.write_all(b"X").unwrap();
15321            let mut child = spawn.await.unwrap();
15322            let early = crate::live_children::read_record(&record).unwrap();
15323            assert_eq!(early.len(), 1);
15324            assert_eq!(early[0].pid, pid);
15325            assert_eq!(
15326                early[0].executable, None,
15327                "unconfirmed trampoline image entered the roster"
15328            );
15329            assert!(child.report_ready.get().is_none());
15330            // The barrier's duration is unrelated to the production five-second
15331            // exec budget. Start the test's confirmation budget upon release.
15332            child.privacy_exec.as_mut().unwrap().deadline =
15333                tokio::time::Instant::now() + Duration::from_secs(30);
15334            exec.write_all(b"X").unwrap();
15335            child.confirm_privacy_exec().await;
15336            assert_eq!(child.spawn_failure, None);
15337            assert!(child.report_ready.get().is_some());
15338            let confirmed = crate::live_children::read_record(&record).unwrap();
15339            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
15340            assert_ne!(module, trampoline);
15341            assert_eq!(confirmed[0].executable, Some(module.into()));
15342            child.start_kill().unwrap();
15343            child.wait().await.unwrap();
15344            child.release_roster();
15345        }
15346    }
15347
15348    #[tokio::test]
15349    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15350    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
15351        #[cfg(target_os = "macos")]
15352        {
15353            let registry = Arc::new(Registry::default());
15354            let policy = RestartPolicy::new(0, Duration::ZERO);
15355            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
15356            let runtime = supervisor.runtime_config();
15357            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15358            let spec = spec("/bin/sh", &["-c", "exit 121"]);
15359            // Drive spawn and confirmation separately instead of starting the
15360            // monitor. WNOWAIT observes a real exit without consuming its status,
15361            // so confirmation's first try_wait must take the already-exited arm.
15362            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
15363            let pid = child.pid;
15364            tokio::task::spawn_blocking(move || {
15365                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
15366            })
15367            .await
15368            .unwrap()
15369            .unwrap();
15370            child.privacy_exec.as_mut().unwrap().deadline =
15371                tokio::time::Instant::now() + Duration::from_secs(30);
15372            let status = child.wait().await.unwrap();
15373            assert_eq!(status.code(), Some(121));
15374            assert!(child.privacy_exec.is_none());
15375            assert!(
15376                child.report_ready.get().is_none(),
15377                "an exited module must not publish a live pid"
15378            );
15379            let report = classify_reaped_child_exit(&snapshot, &child, &status);
15380            on_child_exit(
15381                &spec,
15382                policy,
15383                &registry,
15384                &snapshot,
15385                &runtime.terminal_ring,
15386                &runtime.spawn_events,
15387                &runtime.child_roster,
15388                report,
15389            )
15390            .await;
15391            let state = lock_snapshot(&snapshot).unwrap();
15392            assert_eq!(state.state, ModuleState::Failed);
15393            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
15394            assert_eq!(state.reported_pid(), None);
15395            drop(state);
15396            let history = runtime.terminal_ring.lock().unwrap().snapshot();
15397            assert_eq!(history.entries.len(), 1);
15398            let terminal = &history.entries[0];
15399            assert_eq!(terminal.exit_code, Some(121));
15400            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
15401            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
15402            assert_eq!(
15403                terminal.disposition_detail.as_deref(),
15404                Some(policy.budget_exhausted_detail().as_str()),
15405                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
15406            );
15407            assert_eq!(child.spawn_failure, None);
15408            child.release_roster();
15409        }
15410    }
15411}
15412
15413/// The daemon's real spawn path hands a subc-wire child its launch nonce on
15414/// descriptor 3, without an environment copy. The shell records the nonce
15415/// and its environment after exec so these tests observe the real handover.
15416#[cfg(all(test, unix))]
15417mod launch_nonce_descriptor_tests {
15418    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
15419    use crate::stderr_tail::{StderrRing, StderrTailConfig};
15420    use cortexkit_test_support::ScratchDir;
15421    use std::{
15422        path::PathBuf,
15423        sync::{Arc, Mutex},
15424        time::{Duration, Instant},
15425    };
15426
15427    async fn probe(role: super::SpawnRole) {
15428        let scratch = ScratchDir::new("launch-nonce-descriptor");
15429        let fd_copy = scratch.join("from-descriptor");
15430        let env_copy = scratch.join("environment");
15431        let script = format!(
15432            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
15433            fd = fd_copy.display(), env = env_copy.display(),
15434        );
15435        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
15436        let spec = ModuleSpec {
15437            module_id: "nonce-descriptor-probe".to_string(),
15438            program: PathBuf::from("/bin/sh"),
15439            args: vec!["-c".to_string(), script],
15440            env: vec![
15441                xdg("XDG_DATA_HOME"),
15442                xdg("XDG_RUNTIME_DIR"),
15443                xdg("XDG_CONFIG_HOME"),
15444                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
15445            ],
15446            reserved: true,
15447            reserved_prefixes: Vec::new(),
15448            protocol: ModuleProtocol::Subc,
15449            overlap: Default::default(),
15450        };
15451        let handle = SupervisorHandle::new();
15452        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15453        let roster = ChildRoster::default();
15454        #[cfg(target_os = "macos")]
15455        {
15456            let path = super::test_privacy_trampoline();
15457            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
15458        }
15459        let child = super::spawn_child_in_slot(
15460            &spec,
15461            None,
15462            Some(&handle),
15463            &ring,
15464            None,
15465            &roster,
15466            #[cfg(target_os = "linux")]
15467            None,
15468            role,
15469            matches!(role, super::SpawnRole::SwapCandidate),
15470        )
15471        .expect("spawn probe");
15472        let deadline = Instant::now() + Duration::from_secs(10);
15473        while !(fd_copy.exists() && env_copy.exists()) {
15474            assert!(Instant::now() < deadline, "probe never wrote its copies");
15475            tokio::time::sleep(Duration::from_millis(20)).await;
15476        }
15477        let nonce = std::fs::read_to_string(fd_copy).unwrap();
15478        assert!(!nonce.is_empty());
15479        let environment = std::fs::read_to_string(env_copy).unwrap();
15480        assert!(environment
15481            .lines()
15482            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
15483        let copy = environment
15484            .lines()
15485            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
15486        assert_eq!(
15487            copy, None,
15488            "Unix children must never receive the environment nonce"
15489        );
15490        if matches!(role, super::SpawnRole::Plain) {
15491            assert_eq!(
15492                handle.spawn_nonce(&spec.module_id).as_deref(),
15493                Some(nonce.as_str())
15494            );
15495        }
15496        drop(child);
15497    }
15498
15499    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15500    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
15501        probe(super::SpawnRole::Plain).await;
15502    }
15503
15504    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15505    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
15506        probe(super::SpawnRole::SwapCandidate).await;
15507    }
15508}
15509
15510#[cfg(all(test, target_os = "linux"))]
15511mod cgroup_containment_tests {
15512    use super::*;
15513    use cortexkit_test_support::ScratchDir;
15514
15515    fn running(pid: u32) -> bool {
15516        // An orphan can remain a zombie until the container init reaps it.
15517        std::fs::read_to_string(format!("/proc/{pid}/stat"))
15518            .ok()
15519            .and_then(|stat| {
15520                stat.rsplit_once(") ")
15521                    .map(|(_, rest)| rest.starts_with('Z'))
15522            })
15523            .is_some_and(|zombie| !zombie)
15524    }
15525
15526    #[tokio::test]
15527    async fn linux_teardown_reaps_the_grandchild() {
15528        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
15529    }
15530
15531    #[tokio::test]
15532    async fn linux_shutdown_straggler_reaps_the_grandchild() {
15533        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
15534    }
15535
15536    async fn teardown_tree(test_name: &str, shutdown: bool) {
15537        let dir = ScratchDir::new(test_name);
15538        let root = PathBuf::from(format!(
15539            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
15540            std::process::id(),
15541            unix_ms_now()
15542        ));
15543        if let Err(error) = std::fs::create_dir(&root) {
15544            assert!(
15545                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15546                "required cgroup test cannot execute: {error}"
15547            );
15548            eprintln!(
15549                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
15550                root.display()
15551            );
15552            return;
15553        }
15554        let placement = subc_cgroup::prepare_at(&root)
15555            .expect("prepare isolated kernel cgroup")
15556            .expect("isolated cgroup is delegated");
15557        let module_id = "tree-teardown";
15558        let module = placement
15559            .module_path(module_id)
15560            .expect("create isolated module cgroup");
15561        if !module.join("cgroup.kill").exists() {
15562            std::fs::remove_dir(&module).unwrap();
15563            std::fs::remove_dir(root.join("subc-modules")).unwrap();
15564            std::fs::remove_dir(&root).unwrap();
15565            assert!(
15566                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15567                "required cgroup.kill interface unavailable"
15568            );
15569            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
15570            return;
15571        }
15572        let supervisor = Supervisor::new_for_test(
15573            Arc::new(Registry::default()),
15574            RestartPolicy::new(3, Duration::ZERO),
15575        )
15576        .with_cgroup_placement(Some(placement));
15577        let mut runtime = supervisor.runtime_config();
15578        runtime.child_roster = runtime
15579            .child_roster
15580            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
15581        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15582        let pid_file = dir.join("grandchild.pid");
15583        let spec = ModuleSpec {
15584            module_id: module_id.to_string(),
15585            program: PathBuf::from("/bin/sh"),
15586            args: vec![
15587                "-c".into(),
15588                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
15589                "fixture".into(),
15590                pid_file.display().to_string(),
15591            ],
15592            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15593                .into_iter()
15594                .map(|key| (key.to_string(), dir.display().to_string()))
15595                .collect(),
15596            reserved: false,
15597            reserved_prefixes: Vec::new(),
15598            protocol: ModuleProtocol::None,
15599            overlap: Default::default(),
15600        };
15601        let child =
15602            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
15603        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
15604        let grandchild: u32 = loop {
15605            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
15606                if let Ok(pid) = contents.trim().parse() {
15607                    break pid;
15608                }
15609            }
15610            assert!(
15611                tokio::time::Instant::now() < deadline,
15612                "grandchild pid was not recorded"
15613            );
15614            tokio::time::sleep(Duration::from_millis(10)).await;
15615        };
15616        assert!(
15617            running(grandchild),
15618            "grandchild must be alive before teardown"
15619        );
15620        if shutdown {
15621            let mut child = child;
15622            crate::child_roster::end_children_for_daemon_shutdown(
15623                &runtime.child_roster,
15624                false,
15625                std::future::pending(),
15626            )
15627            .await;
15628            child.wait().await.expect("reap shutdown straggler");
15629        } else {
15630            drain_child_to_state(
15631                module_id,
15632                ModuleProtocol::None,
15633                StopNotice::NotSent,
15634                &Registry::default(),
15635                None,
15636                &snapshot,
15637                &runtime.terminal_ring,
15638                &SpawnEventFeed::default(),
15639                child,
15640                Duration::from_millis(100),
15641                ModuleState::Stopped,
15642                Some(false),
15643            )
15644            .await
15645            .expect("real supervisor teardown");
15646        }
15647        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
15648        while running(grandchild) && tokio::time::Instant::now() < deadline {
15649            tokio::time::sleep(Duration::from_millis(10)).await;
15650        }
15651        let survived = running(grandchild);
15652        // Kill a surviving grandchild so a failed test does not leave it behind.
15653        if survived {
15654            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
15655            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
15656            tokio::time::sleep(Duration::from_millis(100)).await;
15657        }
15658        if module.exists() {
15659            std::fs::remove_dir(&module).expect("remove empty module cgroup");
15660        }
15661        std::fs::remove_dir(root.join("subc-modules")).unwrap();
15662        std::fs::remove_dir(&root).unwrap();
15663        assert!(
15664            !survived,
15665            "grandchild {grandchild} outlived module teardown"
15666        );
15667        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
15668    }
15669}