Skip to main content

subc_daemon/
supervise.rs

1use std::{
2    collections::{HashMap, HashSet, VecDeque},
3    error::Error,
4    fmt, io,
5    path::PathBuf,
6    process::{ExitStatus, Stdio},
7    sync::{Arc, Mutex, OnceLock},
8    time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use subc_control::{
14    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
15    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
16};
17use subc_protocol::{
18    manifest::{SelfSignalKind, SignalAnchor},
19    session::{
20        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
21        MODULE_CONTROL_OP_HEALTH_CHECK,
22    },
23    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
24};
25use tokio::{
26    process::{Child, Command},
27    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
28    task::JoinHandle,
29    time::{sleep, sleep_until, timeout, timeout_at, Instant},
30};
31use tracing::{debug, error, info, warn};
32
33use crate::{
34    child_roster::ChildRoster,
35    daemon_config::{
36        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
37    },
38    forwarding::{
39        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
40        ModuleDrainTarget, PendingModuleControlRpc,
41    },
42    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
43    registry::{ConnectionId, RegistrationEndReason, RegistryError},
44    stderr_tail::{
45        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
46        StderrTailSnapshot,
47    },
48    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
49    Frame, FrameSink, Registry,
50};
51
52#[cfg(not(windows))]
53use crate::provenance::spawned_file_identity;
54
55#[path = "supervise_swap.rs"]
56mod swap;
57
58#[path = "supervise_activity.rs"]
59mod activity;
60
61/// Command-line flag used by supervised modules to find subc.
62///
63/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
64/// The path points at the TCP+key connection file; it is not an ambient signal and
65/// is never inherited by standalone children.
66pub const SUBC_ARG: &str = "--subc";
67
68const DEFAULT_MAX_RESTARTS: u32 = 3;
69const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
70const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
71/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
72/// to contain a real crash loop (which respawns in seconds) and short enough
73/// that unrelated crashes hours apart never accumulate into a permanent stop.
74const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
75/// How long a drain waits for already-dispatched requests to finalize before
76/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
77/// not RPC-scale: the original 2s value silently cut nearly every real tool
78/// call at the fence, making the wait-for-finalize design decorative for the
79/// workloads it existed for. Quiescence short-circuits, so an idle module
80/// restarts immediately regardless of this value; the budget is spent only
81/// when a genuine in-flight request is worth finishing. Per-module override:
82/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
83/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
84/// where a stuck request will never settle).
85pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
86const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
87const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
88/// How long a drain waits after one busy-gauge probe completes before it asks
89/// again.
90///
91/// A drain re-probes on this short interval of its own rather than on the
92/// module's health cadence. The default cadence (30 s) equals the default
93/// drain ceiling, so a cadence-paced drain that found the module busy at its
94/// first probe never asked a second time: it waited out the whole ceiling and
95/// forced GOODBYE even after the gauges had read zero. The interval is counted
96/// from the moment the previous probe finished, and probes run one at a time,
97/// so a slow module is never asked more often than it can answer. Periodic
98/// health checks outside a drain keep using the configured cadence.
99const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
100/// How long a restart waits for the exited process's output readers.
101///
102/// The restart does not depend on either output reader finishing. A stderr
103/// reader still running at this bound is left running, and its later output goes
104/// into the exited process's own section of the stderr ring (see
105/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
106/// bound no longer decides whether a crash's last lines are kept: under load
107/// the reader may simply not have been scheduled yet, and cutting it there
108/// discarded exactly the lines that explained the crash.
109///
110/// What the bound still decides is when the tail starts reporting
111/// `Incomplete`: a pipe open past it usually means a descendant of the exited
112/// process still holds it, and the tail cannot claim to be whole until that
113/// pipe closes. That is also the only case in which the wait costs the restart
114/// anything, because a pipe with no other holder reaches EOF when the process
115/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
116/// `Captured` at EOF with nothing lost.
117///
118/// The stdout reader carries no ring, only the capture file. It also continues
119/// to EOF after this bound: child exit closes the writer but does not mean the
120/// reader has consumed all buffered bytes. Late output from either pipe can
121/// therefore reach the shared capture file after the successor starts.
122const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
123/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
124pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
125const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
126/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
127/// dropped because the subscriber stopped draining its frames. The detail's
128/// `first_undelivered_cursor` names the first event it did not receive; the
129/// client resubscribes from the last cursor it did receive.
130pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
131
132#[cfg(target_os = "macos")]
133struct PrivacyExec {
134    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
135    deadline: tokio::time::Instant,
136    expected: Option<subc_os::FileIdentity>,
137    trampoline: Option<subc_os::FileIdentity>,
138    script: bool,
139    module_id: String,
140    #[cfg(test)]
141    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
142}
143
144#[cfg(all(test, target_os = "macos"))]
145pub(crate) fn test_privacy_trampoline() -> PathBuf {
146    // Cargo's unit-test executable lives in <profile>/deps; its fixture bin
147    // lives beside that directory. This honors custom CARGO_TARGET_DIR too.
148    let path = std::env::current_exe()
149        .unwrap()
150        .parent()
151        .unwrap()
152        .parent()
153        .unwrap()
154        .join("privacy-trampoline-fixture");
155    // Without the fixture every macOS spawn is refused, and the tests that
156    // spawn fail later as a module in state Failed, which names the wrong
157    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
158    assert!(
159        path.exists(),
160        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
161         --bins --features test-support` or `cargo test -p subc-daemon` first",
162        path.display()
163    );
164    path
165}
166
167#[cfg(target_os = "macos")]
168fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
169    use std::io::Read;
170    let mut probe = std::process::Command::new(path)
171        .args(["__disclaim-exec", "--probe"])
172        .stdin(Stdio::null())
173        .stdout(Stdio::piped())
174        .stderr(Stdio::piped())
175        .spawn()
176        .map_err(|error| {
177            format!(
178                "privacy trampoline probe failed for {}: {error}",
179                path.display()
180            )
181        })?;
182    let deadline = std::time::Instant::now() + Duration::from_secs(5);
183    let status = loop {
184        match probe.try_wait() {
185            Ok(Some(status)) => break status,
186            Ok(None) if std::time::Instant::now() < deadline => {
187                std::thread::sleep(Duration::from_millis(5))
188            }
189            result => {
190                let _ = probe.kill();
191                let _ = probe.wait();
192                return Err(format!(
193                    "privacy trampoline probe failed or timed out for {}: {result:?}",
194                    path.display()
195                ));
196            }
197        }
198    };
199    let mut answer = String::new();
200    if let Some(stdout) = probe.stdout.take() {
201        let _ = stdout.take(256).read_to_string(&mut answer);
202    }
203    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
204        return Ok(());
205    }
206    let mut diagnostic = String::new();
207    if let Some(stderr) = probe.stderr.take() {
208        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
209    }
210    let cause = diagnostic
211        .trim()
212        .strip_prefix("ck-subc: own privacy identity refused: ")
213        .unwrap_or("binary does not implement the privacy trampoline protocol");
214    Err(format!(
215        "{cause}: probe of {} exited {status}",
216        path.display()
217    ))
218}
219
220#[cfg(target_os = "macos")]
221fn privacy_command(
222    spec: &ModuleSpec,
223    roster: &ChildRoster,
224) -> Result<
225    (
226        Command,
227        Option<PrivacyExec>,
228        subc_os::privacy_identity::ExecAcknowledgement,
229    ),
230    SuperviseError,
231> {
232    let failure = |cause: String| {
233        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
234        SuperviseError::Spawn {
235            program: spec.program.clone(),
236            source: io::Error::other(cause),
237            cgroup_path: None,
238        }
239    };
240    let trampoline = roster.privacy_trampoline().map_err(failure)?;
241    // Resolve PATH with the same environment the Command will receive. For
242    // scripts retain the existing orphan-identity rule: the kernel chooses
243    // the interpreter, and its observed image is the one recorded. Do not
244    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
245    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
246        let path = spec
247            .env
248            .iter()
249            .find(|(key, _)| key == "PATH")
250            .map(|(_, value)| std::ffi::OsString::from(value))
251            .or_else(|| std::env::var_os("PATH"))
252            .unwrap_or_else(|| "/usr/bin:/bin".into());
253        std::env::split_paths(&path)
254            .map(|dir| dir.join(&spec.program))
255            .find(|path| path.is_file())
256            .unwrap_or_else(|| spec.program.clone())
257    } else {
258        spec.program.clone()
259    };
260    let expected = subc_os::file_identity(&program);
261    let trampoline_image = subc_os::file_identity(&trampoline);
262    let script = {
263        use std::io::Read;
264        let mut prefix = [0u8; 2];
265        std::fs::File::open(&program)
266            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
267    };
268    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
269        return Err(failure(
270            "privacy identity module executable is missing or is the trampoline itself".to_string(),
271        ));
272    }
273    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
274        .map_err(|error| failure(error.to_string()))?;
275    let reader =
276        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
277    let mut command = Command::new(&trampoline);
278    command
279        .arg("__disclaim-exec")
280        .arg(ack.fd().to_string())
281        .arg(&program);
282    ack.install(command.as_std_mut());
283    Ok((
284        command,
285        Some(PrivacyExec {
286            reader,
287            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
288            expected,
289            trampoline: trampoline_image,
290            script,
291            module_id: spec.module_id.clone(),
292            #[cfg(test)]
293            confirmation_pause: spec
294                .env
295                .iter()
296                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
297                .map(|(_, address)| (address.clone(), None)),
298        }),
299        ack,
300    ))
301}
302
303struct SupervisedChild {
304    child: Child,
305    exec: crate::activity::ExecOwner,
306    #[cfg(windows)]
307    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
308    #[cfg(target_os = "macos")]
309    privacy_exec: Option<PrivacyExec>,
310    /// Set once this launch's exec acknowledgement confirms the module image.
311    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
312    /// `subc_os::privacy_identity`), which then replaces itself with the
313    /// module. The supervisor owns and can kill that pid from spawn, but
314    /// status readers report it only after this latch is set, so nothing
315    /// reports the trampoline's image as the module's.
316    #[cfg(target_os = "macos")]
317    report_ready: Arc<OnceLock<()>>,
318    /// Refusal before the module image was accepted, retained for terminal records.
319    spawn_failure: Option<String>,
320    /// The protocol this process was launched with. A reload can store a new
321    /// launch spec with a different protocol, but that takes effect only at the
322    /// next spawn, so this process keeps being handled by the protocol it
323    /// actually speaks.
324    protocol: ModuleProtocol,
325    /// This process's cgroup name: a bounded module/slot label followed by a
326    /// spawn suffix unique to this process (when cgroup placement is on). A
327    /// retired process in a slot may still be draining when a later one is
328    /// spawned into that slot, so the suffix keeps the later process out of
329    /// the retired one's cgroup, which is the domain a kill applies to.
330    #[cfg(target_os = "linux")]
331    module_id: String,
332    #[cfg(target_os = "linux")]
333    cgroup_placement: Option<subc_cgroup::Placement>,
334    /// The job that contains this child and every process it spawns (issue #109).
335    ///
336    /// Dropping this handle is what reaps a surviving tree when no supervisor
337    /// code runs — a daemon crash — because the job carries
338    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
339    ///
340    /// That limit is not crash-only, and the difference is worth knowing: a
341    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
342    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
343    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
344    /// module at once. Before this change they survived that, saw EOF on the
345    /// control socket, and ran their own teardown; Unix keeps that path
346    /// deliberately, so a module can seal a WAL or close a capture rather than
347    /// be killed mid-write. So this trades graceful teardown on every Windows
348    /// daemon stop for containment on a crash, which is the right way round
349    /// today: orphaned GPU workers are a reported, recurring problem, and the
350    /// modules that write most heavily do not run on Windows.
351    ///
352    /// The fix is a real Windows stop path — the daemon draining before it
353    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
354    /// reaches only what the drain left behind, which is what it should reach.
355    #[cfg(windows)]
356    job: Option<subc_jobobject::JobObject>,
357    stdout_pump: Option<JoinHandle<()>>,
358    stderr_pump: Option<StderrPump>,
359    stderr_ring: Arc<Mutex<StderrRing>>,
360    spawned_at_ms: u64,
361    spawned_from: PathBuf,
362    spawned_file_identity: Option<SpawnedFileIdentity>,
363    process_start_time: Option<u64>,
364    process_identity: Option<ProcessIdentity>,
365    pid: u32,
366    /// This process's entry in the daemon's child roster, released when the
367    /// process is reaped or this handle is dropped.
368    roster_guard: Option<crate::child_roster::RosterGuard>,
369}
370
371impl SupervisedChild {
372    fn id(&self) -> Option<u32> {
373        Some(self.pid)
374    }
375
376    fn process_identity(&self) -> Option<ProcessIdentity> {
377        self.process_identity
378    }
379
380    async fn wait(&mut self) -> io::Result<ExitStatus> {
381        #[cfg(target_os = "macos")]
382        self.confirm_privacy_exec().await;
383        // The roster entry is NOT released here. A daemon shutdown waits for the
384        // roster to empty and then exits the process, so releasing at the reap
385        // let it exit before the exit handler wrote this child's terminal record
386        // (the stderr drain and snapshot update sit in between), and the
387        // shutdown's own `daemon_shutdown` record was intermittently lost. The
388        // caller releases it after recording the exit (`release_roster`), and
389        // dropping the handle releases it too.
390        let result = self.child.wait().await;
391        #[cfg(target_os = "linux")]
392        if result.is_ok() {
393            if let Some(placement) = self.cgroup_placement.as_ref() {
394                cleanup_reaped_cgroup(placement, &self.module_id).await;
395                // Keep ownership while awaiting kernel population changes: a
396                // drain timeout may cancel this wait and then escalate/reap.
397                self.cgroup_placement = None;
398            }
399        }
400        result
401    }
402
403    #[cfg(target_os = "macos")]
404    async fn confirm_privacy_exec(&mut self) {
405        let Some(pending) = &mut self.privacy_exec else {
406            return;
407        };
408        let result = tokio::time::timeout_at(pending.deadline, async {
409            let mut record = Vec::new();
410            loop {
411                let mut ready = pending.reader.readable().await?;
412                let read = ready.try_io(|reader| {
413                    use std::io::Read;
414                    let mut reader = reader.get_ref();
415                    let mut buffer = [0u8; 256];
416                    reader.read(&mut buffer).map(|count| (count, buffer))
417                });
418                match read {
419                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
420                    Ok(Ok((count, buffer))) => {
421                        if record.len() + count > 1024 {
422                            return Err(io::Error::other(
423                                "privacy exec refusal record is too long",
424                            ));
425                        }
426                        record.extend_from_slice(&buffer[..count]);
427                    }
428                    Ok(Err(error)) => return Err(error),
429                    Err(_) => continue,
430                }
431            }
432        })
433        .await;
434        #[cfg(test)]
435        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
436            use tokio::io::{AsyncReadExt, AsyncWriteExt};
437            if stream.is_none() {
438                *stream = Some(
439                    tokio::net::TcpStream::connect(address.as_str())
440                        .await
441                        .unwrap(),
442                );
443                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
444                stream
445                    .as_mut()
446                    .unwrap()
447                    .write_all(&self.pid.to_ne_bytes())
448                    .await
449                    .unwrap();
450            }
451            let mut reply = [0];
452            stream
453                .as_mut()
454                .unwrap()
455                .read_exact(&mut reply)
456                .await
457                .unwrap();
458            pending.confirmation_pause = None;
459            if &reply == b"F" {
460                Ok(Err(io::Error::other("injected late exec refusal")))
461            } else {
462                assert_eq!(&reply, b"X");
463                result
464            }
465        } else {
466            result
467        };
468        // Keep the reader in self across await: select cancellation must not
469        // discard the handshake or reset its original five-second deadline.
470        let pending = self.privacy_exec.as_ref().expect("pending exec");
471        let cause = match result {
472            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
473            Ok(Err(error)) => Some(format!(
474                "privacy identity exec acknowledgement failed: {error}"
475            )),
476            Ok(Ok(record)) if !record.is_empty() => Some(
477                std::str::from_utf8(&record)
478                    .ok()
479                    .and_then(|record| {
480                        record
481                            .trim()
482                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
483                    })
484                    .filter(|cause| !cause.is_empty())
485                    .unwrap_or("invalid privacy exec refusal record")
486                    .to_string(),
487            ),
488            Ok(Ok(_)) => match self.child.try_wait() {
489                // Empty EOF is the exec acknowledgement. A real module may exit
490                // immediately, including with a reserved trampoline status; no
491                // image is admitted, and its ordinary exit contract stays intact.
492                Ok(Some(_status)) => None,
493                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
494                Ok(None) => {
495                    let image = observe_spawned_image(self.pid);
496                    if let Some(image) = image.filter(|image| {
497                        image.executable.is_some()
498                            && image.executable != pending.trampoline
499                            && (image.executable == pending.expected || pending.script)
500                    }) {
501                        if let Err(error) = self
502                            .roster_guard
503                            .as_ref()
504                            .map(|guard| guard.confirm_executable(image))
505                            .transpose()
506                        {
507                            Some(format!(
508                                "could not durably record confirmed module image: {error}"
509                            ))
510                        } else {
511                            let _ = self.report_ready.set(());
512                            info!(module_id = %pending.module_id, pid = self.pid,
513                                "module spawned with own privacy identity (responsibility disclaimed)");
514                            None
515                        }
516                    } else if image.is_none()
517                        || image.is_some_and(|image| image.executable.is_none())
518                    {
519                        // A process can exit between try_wait and the kernel
520                        // image read. Empty EOF already acknowledged exec, so
521                        // preserve that module's ordinary exit rather than
522                        // mislabel a disappearing image as trampoline refusal.
523                        // Keep pending in self across await so cancellation does
524                        // not discard validation or reset its original deadline.
525                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
526                            Ok(Ok(_status)) => None,
527                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
528                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
529                        }
530                    } else {
531                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
532                    }
533                }
534            },
535        };
536        let pending = self.privacy_exec.take().expect("pending exec");
537        // Exec confirmation describes launch success, not roster retirement.
538        // Keep even a reaped child recorded until its terminal history exists.
539        self.exec.0.finish(if cause.is_some() {
540            crate::activity::ExecState::Refused
541        } else {
542            crate::activity::ExecState::CommittedExec
543        });
544        if let Some(cause) = cause {
545            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
546            self.spawn_failure = Some(cause);
547            // No image is admitted on failure. Reach the entire fresh process
548            // group, including a module which spawned a helper before refusal.
549            if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
550                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
551            }
552            let _ = self.child.start_kill();
553        }
554    }
555
556    /// Releases this child's daemon-shutdown roster entry once its exit has
557    /// been recorded. The pid is already reaped and free for reuse, so the
558    /// entry must not outlive the record any longer than that.
559    fn release_roster(&mut self) {
560        self.roster_guard = None;
561    }
562
563    /// Kill the child and, where containment is available, its process tree.
564    ///
565    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
566    /// helper process leaked the helper — the Synapse embedding module's CUDA
567    /// worker holds the GPU allocation, so the leak cost VRAM until the next
568    /// restart of something else. Terminating the job reaches grandchildren that
569    /// a tree walk cannot, including one whose parent has already exited and
570    /// been reparented away.
571    ///
572    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
573    /// direct-child kill still decides the outcome, so containment can never
574    /// change whether a module is reported as stopped.
575    fn start_kill(&mut self) -> io::Result<()> {
576        #[cfg(windows)]
577        if let Some(job) = &self.job {
578            if let Err(error) = job.terminate() {
579                debug!(
580                    error = %error,
581                    "job termination failed; the direct-child kill still owns the outcome"
582                );
583            }
584        }
585        #[cfg(target_os = "linux")]
586        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
587        self.child.start_kill()
588    }
589
590    async fn drain_stderr(&mut self, module_id: &str) {
591        if let Some(pump) = self.stdout_pump.take() {
592            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
593        }
594
595        let Some(pump) = self.stderr_pump.take() else {
596            return;
597        };
598        settle_stderr_pump(
599            module_id,
600            &self.stderr_ring,
601            pump,
602            STDERR_PUMP_DRAIN_TIMEOUT,
603        )
604        .await;
605    }
606}
607
608/// Bound the restart's wait, not the lifetime of the exited child's stdout
609/// reader. Slow pipe reads or capture writes must not turn the bound into lost
610/// output; dropping the handle lets the reader finish at EOF, like stderr.
611async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
612    match timeout(bound, &mut pump).await {
613        Ok(Ok(())) => {}
614        Ok(Err(error)) => {
615            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
616        }
617        Err(_) => {
618            drop(pump);
619            warn!(
620                module_id,
621                waited = ?bound,
622                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
623            );
624        }
625    }
626}
627
628/// The reader task for one process's stderr, with the ring generation its
629/// lines are attributed to.
630struct StderrPump {
631    task: JoinHandle<()>,
632    generation: u64,
633}
634
635/// Retire an exited process's stderr reader and wait up to `bound` for it to
636/// reach EOF. A reader still running at the bound is detached, not stopped: it
637/// keeps filling the exited process's section of the ring until its pipe
638/// closes, and the tail reads `Incomplete` until then. See
639/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
640async fn settle_stderr_pump(
641    module_id: &str,
642    ring: &Arc<Mutex<StderrRing>>,
643    pump: StderrPump,
644    bound: Duration,
645) {
646    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
647    let StderrPump {
648        mut task,
649        generation,
650    } = pump;
651    lock().retire_pump(generation);
652    match timeout(bound, &mut task).await {
653        Ok(Ok(())) => {}
654        Ok(Err(err)) => {
655            let mut ring = lock();
656            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
657            ring.finish_pump(generation);
658            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
659        }
660        Err(_) => {
661            // Dropping the handle detaches the task; it ends at EOF on its pipe.
662            drop(task);
663            lock().mark_pump_late(
664                generation,
665                format!(
666                    "stderr of the exited process had not reached EOF {bound:?} after it was \
667                     retired (a descendant may still hold the pipe open); lines it still \
668                     writes are kept in that process's section"
669                ),
670            );
671            warn!(
672                module_id,
673                waited = ?bound,
674                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
675            );
676        }
677    }
678}
679
680fn registration_release_events() -> &'static watch::Sender<u64> {
681    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
682    EVENTS.get_or_init(|| {
683        let (sender, _receiver) = watch::channel(0);
684        sender
685    })
686}
687
688pub(crate) fn notify_registration_release() {
689    let events = registration_release_events();
690    let next_generation = (*events.borrow()).wrapping_add(1);
691    events.send_replace(next_generation);
692}
693
694/// How to launch one singleton module process.
695#[derive(Debug, Clone, PartialEq, Eq)]
696pub struct ModuleSpec {
697    pub module_id: String,
698    pub program: PathBuf,
699    pub args: Vec<String>,
700    pub env: Vec<(String, String)>,
701    /// When true this is a reserved module: each spawn gets a fresh one-time launch
702    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
703    /// process can register this module_id (a security-boundary module like the
704    /// credential vault must not be impersonable while it is down/restarting).
705    pub reserved: bool,
706    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
707    /// Prefixes come from daemon config and must end in `:` before they reach the
708    /// supervisor; the owner module's current spawn nonce authorizes claims under
709    /// each prefix.
710    pub reserved_prefixes: Vec<String>,
711    /// The wire protocol this module speaks, as DECLARED in daemon config.
712    ///
713    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
714    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
715    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
716    /// and NO launch nonce, and a clean exit the daemon did not request is
717    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
718    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
719    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
720    /// because a process ignores an environment variable it does not read.
721    ///
722    /// The argument is the part that cannot be "harmless to a process that
723    /// ignores it": a stock binary exits on an unknown flag before it listens
724    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
725    /// first conformance run against this mode found it. The nonce is withheld
726    /// because a process that will never present it gains nothing from holding
727    /// it, and a secret in the environment of a process that does not need it is
728    /// a leak surface for no benefit.
729    pub protocol: ModuleProtocol,
730    /// Whether two processes of this module may run at once, which is what a
731    /// blue/green swap does for the length of its overlap. Declared in daemon
732    /// config because the daemon must be able to answer it while the module is
733    /// down, and so a module cannot talk itself into it after registering.
734    pub overlap: ModuleOverlap,
735}
736
737/// Whether a module tolerates a second process of itself running alongside.
738///
739/// Most modules are single-writer on their store (a WAL, a capture log, a
740/// resident index behind a writer barrier), and two processes on one store
741/// corrupt it. So a swap, which overlaps the old and new process by design,
742/// is refused unless the module's config opts in with `overlap: "safe"`.
743#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
744pub enum ModuleOverlap {
745    /// Never run two processes of this module at once. The default.
746    #[default]
747    Exclusive,
748    /// The module has said a second process of itself is harmless for the
749    /// length of a swap.
750    ///
751    /// Declare it only if a second instance can run for a few seconds without
752    /// touching ANY single-writer store: every database, WAL, index, projector
753    /// and scheduled job the module owns. A lease on part of that state is not
754    /// enough. broca's session lease guards WAL appends while its run index, its
755    /// store projector and its archive fold timer (which unlinks live WAL files)
756    /// stay single-writer, so broca is exclusive despite holding a lease. The
757    /// refusal only fires after this has been decided, so the decision is the
758    /// check.
759    Safe,
760}
761
762impl ModuleOverlap {
763    pub fn as_str(self) -> &'static str {
764        match self {
765            Self::Exclusive => "exclusive",
766            Self::Safe => "safe",
767        }
768    }
769}
770
771/// Environment variable telling a spawned module which case it was started
772/// for, before it sends HELLO. Only a swap candidate carries it, as
773/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
774///
775/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
776/// longer because nobody waits on it, while a plain restart must flip ready
777/// quickly because callers see `module_warming` until it does. Absence means
778/// plain restart, the safe reading. The daemon trusts nothing about it; the
779/// candidate is proven by its launch nonce at HELLO.
780pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
781/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
782pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
783/// How long a swap waits for its candidate to register and declare itself
784/// ready when the operator does not say. A module warming as a swap candidate
785/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
786/// daemon allows that plus time to start the process and send HELLO.
787pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
788
789/// Bounded restart policy for crash exits.
790///
791/// `max_restarts` is the number of replacement processes allowed after the
792/// initial spawn WITHIN `window`. After that many crash restarts inside one
793/// window the module enters [`ModuleState::Failed`] and the supervisor stops
794/// the crash loop.
795///
796/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
797/// and that only survived because crashes were rare: a module that crashed
798/// three times across a week was disabled forever by crashes that had nothing
799/// to do with each other. That stopped being survivable once modules began
800/// exiting non-zero whenever the daemon's connection to them drops, because
801/// then every daemon-side connection drop spends a unit of the same budget and
802/// one flappy hour permanently stops a healthy module. Restarts older than
803/// `window` release their slot, so a module that crashed twice yesterday has a
804/// full budget today, while a genuine crash loop -- which is fast by
805/// definition -- still reaches the cap and stops.
806#[derive(Debug, Clone, Copy, PartialEq, Eq)]
807pub struct RestartPolicy {
808    pub max_restarts: u32,
809    /// Base delay before a crash replacement. The actual delay escalates with
810    /// the number of recent crash replacements and is capped by `max_backoff`.
811    pub backoff: Duration,
812    /// Maximum delay before a crash replacement.
813    pub max_backoff: Duration,
814    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
815    /// budget effectively infinite (nothing is ever in-window), which is why
816    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
817    pub window: Duration,
818}
819
820impl RestartPolicy {
821    /// A policy with the default crash window. Callers that care about the
822    /// window say so with [`Self::with_window`]; the ones that do not are
823    /// asking for the standard rate limit, not for no limit.
824    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
825        Self {
826            max_restarts,
827            backoff,
828            max_backoff: DEFAULT_MAX_BACKOFF,
829            window: DEFAULT_RESTART_WINDOW,
830        }
831    }
832
833    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
834        self.max_backoff = max_backoff;
835        self
836    }
837
838    pub fn with_window(mut self, window: Duration) -> Self {
839        self.window = window;
840        self
841    }
842
843    /// Calculate the capped exponential delay for the next crash replacement.
844    /// `restart_in_window` is zero for the first replacement after an operator
845    /// action (restart, reload, re-enable) cleared the crash ring, or after all
846    /// older crash replacements have aged out of the window.
847    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
848        if self.backoff.is_zero() || self.max_backoff.is_zero() {
849            return Duration::ZERO;
850        }
851
852        let mut delay = self.backoff;
853        for _ in 0..restart_in_window {
854            if delay >= self.max_backoff {
855                return self.max_backoff;
856            }
857            delay = delay
858                .checked_mul(10)
859                .unwrap_or(self.max_backoff)
860                .min(self.max_backoff);
861        }
862        delay.min(self.max_backoff)
863    }
864
865    /// The one sentence that explains a budget-exhausted stop, used for both the
866    /// log line and the terminal record so the two cannot drift. It names the
867    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
868    /// exactly what this budget is not.
869    fn budget_exhausted_detail(&self) -> String {
870        format!(
871            "crash budget exhausted: max_restarts={} within window_secs={}",
872            self.max_restarts,
873            self.window.as_secs()
874        )
875    }
876}
877
878impl Default for RestartPolicy {
879    fn default() -> Self {
880        Self {
881            max_restarts: DEFAULT_MAX_RESTARTS,
882            backoff: DEFAULT_BACKOFF,
883            max_backoff: DEFAULT_MAX_BACKOFF,
884            window: DEFAULT_RESTART_WINDOW,
885        }
886    }
887}
888
889#[derive(Debug, Clone, Copy, PartialEq, Eq)]
890struct CrashRestartSchedule {
891    restart_in_window: u32,
892    delay: Duration,
893}
894
895/// Whether the daemon itself will bring this module back after the exit being
896/// handled: it is enabled AND its in-window crash restarts are below the cap.
897///
898/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
899/// the window are dropped here rather than by a timer, so the count is right
900/// the moment somebody asks and no bookkeeping runs for idle modules.
901fn daemon_will_restart(
902    state: &mut SupervisorSnapshot,
903    policy: &RestartPolicy,
904    now: Instant,
905) -> bool {
906    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
907}
908
909const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
910const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
911const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
912const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
913
914#[derive(Debug, Clone, Copy, PartialEq, Eq)]
915pub enum HealthAction {
916    Report,
917    Restart,
918    Alert,
919}
920
921impl fmt::Display for HealthAction {
922    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
923        f.write_str(match self {
924            Self::Report => "report",
925            Self::Restart => "restart",
926            Self::Alert => "alert",
927        })
928    }
929}
930
931#[derive(Debug, Clone, PartialEq, Eq)]
932pub struct HealthConfig {
933    /// Optional loopback HTTP endpoint for a managed non-wire process.
934    /// Changing it applies live on rescan; the process protocol changes only
935    /// at its next spawn.
936    pub http: Option<String>,
937    pub cadence: Duration,
938    pub deadline: Duration,
939    pub failure_threshold: u32,
940    pub on_degraded: HealthAction,
941    pub on_failing: HealthAction,
942    pub critical: bool,
943}
944
945impl Default for HealthConfig {
946    fn default() -> Self {
947        Self {
948            http: None,
949            cadence: DEFAULT_HEALTH_CADENCE,
950            deadline: DEFAULT_HEALTH_DEADLINE,
951            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
952            on_degraded: HealthAction::Report,
953            on_failing: HealthAction::Report,
954            critical: false,
955        }
956    }
957}
958
959/// The supervisor's view of one module's health, relayed to clients over
960/// channel-0 and rendered by `ck health`.
961///
962/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
963/// stated here rather than only at the wire type a consumer reads. A reader can
964/// look up what `None` means; only a writer can silently change it, and the
965/// writer has no reason to go looking at a downstream contract before editing.
966///
967/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
968/// back to `None` on re-registration precisely so a respawned module does not
969/// carry its predecessor's timestamp — so an old value and an absent one call for
970/// opposite readings, and anything that defaulted this to a number would make a
971/// never-probed module indistinguishable from one probed at the epoch.
972///
973/// `detail` and `metrics` are `None` when the module published none on this
974/// probe, which does not mean it reported nothing wrong — it is also the shape
975/// when the probe never reached it. `last_probe_ms` is what separates those.
976#[derive(Debug, Clone, PartialEq)]
977pub struct ModuleHealthStatus {
978    pub status: SupervisorHealthStatus,
979    pub last_probe_ms: Option<u64>,
980    pub detail: Option<String>,
981    pub metrics: Option<Value>,
982    pub consecutive_failures: u32,
983    /// Number of replies received after a recurring health probe's deadline.
984    /// Unlike a timeout, every increment proves the module was alive.
985    pub late_answer_count: u64,
986    /// End-to-end latency of the newest late reply, measured from probe start.
987    pub last_late_answer_latency_ms: Option<u64>,
988    pub last_action: Option<String>,
989    /// Set together with `last_action`; the pair moves as one, and both being
990    /// absent means no escalation has ever been taken rather than that the last
991    /// one succeeded.
992    pub last_action_ms: Option<u64>,
993}
994
995impl Default for ModuleHealthStatus {
996    fn default() -> Self {
997        Self {
998            status: SupervisorHealthStatus::Unknown,
999            last_probe_ms: None,
1000            detail: None,
1001            metrics: None,
1002            consecutive_failures: 0,
1003            late_answer_count: 0,
1004            last_late_answer_latency_ms: None,
1005            last_action: None,
1006            last_action_ms: None,
1007        }
1008    }
1009}
1010
1011/// Typed lifecycle state for a supervised module.
1012#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1013pub enum ModuleState {
1014    Starting,
1015    Running,
1016    Unresponsive,
1017    Restarting,
1018    Draining,
1019    Stopped,
1020    Failed,
1021    Disabled,
1022}
1023
1024impl fmt::Display for ModuleState {
1025    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1026        f.write_str(match self {
1027            Self::Starting => "starting",
1028            Self::Running => "running",
1029            Self::Unresponsive => "unresponsive",
1030            Self::Restarting => "restarting",
1031            Self::Draining => "draining",
1032            Self::Stopped => "stopped",
1033            Self::Failed => "failed",
1034            Self::Disabled => "disabled",
1035        })
1036    }
1037}
1038
1039/// Supervisor classification of a child-process exit.
1040#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1041pub enum ExitKind {
1042    Clean,
1043    Crash,
1044    DeliberateSeverance,
1045}
1046
1047impl From<ExitKind> for TerminalExitKind {
1048    fn from(kind: ExitKind) -> Self {
1049        match kind {
1050            ExitKind::Clean => Self::Clean,
1051            ExitKind::Crash => Self::Crash,
1052            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1053        }
1054    }
1055}
1056
1057/// Exact process identity retained when a supervised module registers its
1058/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1059#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1060pub(crate) struct ProcessIdentity {
1061    pub(crate) pid: u32,
1062    pub(crate) start_time: u64,
1063}
1064
1065/// Last observed child exit, if any.
1066#[derive(Debug, Clone, PartialEq, Eq)]
1067pub struct ExitReport {
1068    pub kind: ExitKind,
1069    pub code: Option<i32>,
1070    pub signal: Option<i32>,
1071    pub at_ms: u64,
1072}
1073
1074/// Point-in-time module status answerable by subc without forwarding to the
1075/// module process.
1076#[derive(Debug, Clone, PartialEq)]
1077pub struct ModuleStatus {
1078    pub module_id: String,
1079    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1080    pub waiting_for_previous_pid: Option<u32>,
1081    /// Why the previous instance still prevents a new start.
1082    pub waiting_for_previous_detail: Option<String>,
1083    /// The daemon's Windows per-spawn record: `pipe` after the process it
1084    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1085    /// List/status reports use this record, not the source the module sends
1086    /// in its registration message's manifest.provenance.launch_nonce_source.
1087    pub launch_nonce_source: Option<String>,
1088    pub state: ModuleState,
1089    pub enabled: bool,
1090    pub process_alive: bool,
1091    pub registration_active: bool,
1092    /// The module's declared wire protocol, carried beside `live` because it is
1093    /// what makes `live` readable: the two fields answer one question together.
1094    /// While a process is alive this is its launch declaration, not a later
1095    /// pending-reload edit. When down it is the configured next launch protocol.
1096    pub protocol: ModuleProtocol,
1097    /// Whether the module is serving, under the strongest definition the daemon
1098    /// can assert for its protocol.
1099    ///
1100    /// A subc module must also be REGISTERED: its process being alive says
1101    /// nothing about whether it can take a request. A `protocol: "none"` module
1102    /// never registers, so that term is dropped and this falls back to "enabled,
1103    /// running, and the process the daemon launched is alive" -- which is all
1104    /// the daemon observes about a process that speaks no subc wire. It stays a
1105    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1106    /// rather than printing it bare.
1107    pub live: bool,
1108    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1109    /// restarts have already released their slot, so this count can go down
1110    /// without anybody touching the module.
1111    pub restart_count: u32,
1112    /// Replacement processes spawned over this module's entire supervisor lifetime;
1113    /// unlike `restart_count`, this value is never reset by an operator action
1114    /// and never falls out of a window.
1115    pub lifetime_restarts: u32,
1116    pub spawn_generation: u64,
1117    /// The budget `restart_count` is spent against. Carried alongside the count
1118    /// because the count alone does not say how close the module is to being
1119    /// disabled, and reporting one without the other is what makes an
1120    /// about-to-be-retired module look ordinary.
1121    pub max_restarts: u32,
1122    /// The span `restart_count` is counted over. Carried with the pair above for
1123    /// the same reason they are carried together: "2 of 3" means one thing for a
1124    /// ten-minute window and something else entirely for a lifetime.
1125    pub restart_window: Duration,
1126    /// Effective drain and restart timing policy used by this running module.
1127    /// These values are carried together with the restart budget so status
1128    /// readers can compare configured intent with what the supervisor applied.
1129    pub drain_timeout: Duration,
1130    pub restart_backoff: Duration,
1131    pub restart_max_backoff: Duration,
1132    /// The module's process. On macOS this stays absent while the `ck-subc`
1133    /// launch trampoline is still running in that pid, and appears once the
1134    /// exec acknowledgement confirms the module image has replaced it. Launch
1135    /// time and the supervisor's own hold on the process are unaffected.
1136    pub pid: Option<u32>,
1137    pub spawned_at_ms: Option<u64>,
1138    pub spawned_from: Option<PathBuf>,
1139    pub process_start_time: Option<u64>,
1140    pub last_exit: Option<ExitReport>,
1141    pub health: ModuleHealthStatus,
1142}
1143
1144#[cfg(test)]
1145#[derive(Debug, Clone, Copy, PartialEq)]
1146struct ActorSelectCheckpoint {
1147    generation: u64,
1148    turn: u64,
1149    registered_connection: Option<ConnectionId>,
1150    next_probe_at: Option<Instant>,
1151    wake_after: Option<Duration>,
1152}
1153
1154#[derive(Debug, Clone, PartialEq)]
1155struct SupervisorSnapshot {
1156    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1157    #[cfg(windows)]
1158    pipe_consumed: Option<Arc<OnceLock<()>>>,
1159    /// Counts running-child loop turns, not executor polls of a parked wait.
1160    #[cfg(test)]
1161    actor_turns: u64,
1162    /// Running select after exec confirmation and with no unconsumed registry
1163    /// event. Recording a spawn or starting a loop turn is not this barrier.
1164    #[cfg(test)]
1165    actor_select: Option<ActorSelectCheckpoint>,
1166    state: ModuleState,
1167    enabled: bool,
1168    process_alive: bool,
1169    spawned_protocol: Option<ModuleProtocol>,
1170    spawn_failure: Option<String>,
1171    waiting_for_previous_pid: Option<u32>,
1172    waiting_for_previous_detail: Option<String>,
1173    /// When each crash restart was spent, oldest first. This IS the crash
1174    /// budget: its in-window length is the count an operator sees and the count
1175    /// the restart decision is made against, so there is no second counter that
1176    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1177    /// operator actions that used to zero the old lifetime counter.
1178    crash_restarts: VecDeque<Instant>,
1179    lifetime_restarts: u32,
1180    /// Successful child spawns in this daemon incarnation.
1181    ///
1182    /// `lifetime_restarts` was considered and rejected: it starts at zero
1183    /// (line 640), successful initial/operator spawns in `set_running` do not
1184    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1185    /// increments before a successful replacement exists (lines 604, 3846,
1186    /// and 3921), so a failed spawn can consume it. This counter moves only
1187    /// when a live PID is accepted below.
1188    spawn_generation: u64,
1189    pid: Option<u32>,
1190    #[cfg(target_os = "macos")]
1191    report_ready: Option<Arc<OnceLock<()>>>,
1192    /// Last reaped child, retained after current process facts are cleared.
1193    reaped_pid: Option<u32>,
1194    /// Whether a replacement has already been scheduled for the current process.
1195    respawn_pending: bool,
1196    /// A second restart is waiting for the replacement already scheduled.
1197    coalesced_restart_pending: bool,
1198    spawned_at_ms: Option<u64>,
1199    spawned_from: Option<PathBuf>,
1200    spawned_file_identity: Option<SpawnedFileIdentity>,
1201    process_start_time: Option<u64>,
1202    deliberate_severance: Option<ProcessIdentity>,
1203    last_exit: Option<ExitReport>,
1204    /// Diagnostic attached to the next drain's terminal record, if any.
1205    drain_disposition_detail: Option<String>,
1206    health: ModuleHealthStatus,
1207    /// Whether the current process was started as a swap candidate and so
1208    /// lives in the module's alternate cgroup. The next swap's candidate takes
1209    /// the other one, so the two processes of a swap never share a cgroup. A
1210    /// plain spawn always uses the primary cgroup.
1211    in_alternate_slot: bool,
1212    /// Whether the current `Draining` state ends in a replacement process
1213    /// (restart, reload, health restart) rather than a stop. Only meaningful
1214    /// while `state` is `Draining`; every entry into that state rewrites it.
1215    /// It is what lets route.open answer the retryable `module_reloading` to a
1216    /// consumer that reaches a still-registered process mid-restart, instead of
1217    /// the `supervisor_not_live` a stop or disable deserves.
1218    draining_to_replace: bool,
1219    /// Whether a configuration update has been applied since the current
1220    /// process was spawned, so that process runs an older spec than the one
1221    /// the supervisor now holds. A queued restart is only coalesced into a
1222    /// fresher process when this is false: a restart requested to pick up a
1223    /// new configuration must not be satisfied by a process that predates it.
1224    configuration_updated_since_spawn: bool,
1225}
1226
1227impl SupervisorSnapshot {
1228    /// The pid that status, provenance and resource readings may report. While
1229    /// the launch trampoline still runs in the pid, reading its executable or
1230    /// resource use would describe `ck-subc`, not the module, so none is
1231    /// reported until the exec acknowledgement confirms the module image.
1232    fn reported_pid(&self) -> Option<u32> {
1233        #[cfg(target_os = "macos")]
1234        if self
1235            .report_ready
1236            .as_ref()
1237            .is_some_and(|ready| ready.get().is_none())
1238        {
1239            return None;
1240        }
1241        self.pid
1242    }
1243
1244    fn starting() -> Self {
1245        Self::new(ModuleState::Starting, true)
1246    }
1247
1248    fn disabled() -> Self {
1249        Self::new(ModuleState::Disabled, false)
1250    }
1251
1252    /// Crash restarts still inside `window`, having dropped the ones that are
1253    /// not. Pruning on read is what makes the budget a rate: an instant older
1254    /// than the window stops holding a slot the moment anybody counts.
1255    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1256        while let Some(oldest) = self.crash_restarts.front() {
1257            if now.duration_since(*oldest) > window {
1258                self.crash_restarts.pop_front();
1259            } else {
1260                break;
1261            }
1262        }
1263        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1264    }
1265
1266    /// Spend one unit of the crash budget and record the restart in the ledger.
1267    ///
1268    /// The ring is bounded by the cap because more than `max_restarts` in-window
1269    /// instants can never be reached (the caller refuses the restart first), so
1270    /// anything beyond that is an unbounded queue waiting to happen.
1271    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1272        self.crash_restarts.push_back(now);
1273        while self.crash_restarts.len() > policy.max_restarts as usize {
1274            self.crash_restarts.pop_front();
1275        }
1276        self.lifetime_restarts += 1;
1277    }
1278
1279    /// Reserve one crash-restart slot and calculate the delay before respawning.
1280    /// The count is captured before recording this restart, so the first retry
1281    /// uses the base delay and each later in-window retry escalates once.
1282    fn next_crash_restart(
1283        &mut self,
1284        policy: &RestartPolicy,
1285        now: Instant,
1286    ) -> Option<CrashRestartSchedule> {
1287        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1288        if restart_in_window >= policy.max_restarts {
1289            return None;
1290        }
1291        self.record_crash_restart(policy, now);
1292        Some(CrashRestartSchedule {
1293            restart_in_window,
1294            delay: policy.delay_for_restart(restart_in_window),
1295        })
1296    }
1297
1298    /// Give the module its full budget back, as an operator restart, reload, or
1299    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1300    /// ledger of what actually happened, and an operator action does not unmake
1301    /// the crashes.
1302    fn clear_crash_restarts(&mut self) {
1303        self.crash_restarts.clear();
1304    }
1305
1306    fn new(state: ModuleState, enabled: bool) -> Self {
1307        Self {
1308            exec_outcome: None,
1309            #[cfg(test)]
1310            actor_turns: 0,
1311            #[cfg(test)]
1312            actor_select: None,
1313            state,
1314            enabled,
1315            process_alive: false,
1316            #[cfg(windows)]
1317            pipe_consumed: None,
1318            spawned_protocol: None,
1319            spawn_failure: None,
1320            waiting_for_previous_pid: None,
1321            waiting_for_previous_detail: None,
1322            crash_restarts: VecDeque::new(),
1323            lifetime_restarts: 0,
1324            spawn_generation: 0,
1325            pid: None,
1326            #[cfg(target_os = "macos")]
1327            report_ready: None,
1328            reaped_pid: None,
1329            respawn_pending: false,
1330            coalesced_restart_pending: false,
1331            spawned_at_ms: None,
1332            spawned_from: None,
1333            spawned_file_identity: None,
1334            process_start_time: None,
1335            deliberate_severance: None,
1336            last_exit: None,
1337            drain_disposition_detail: None,
1338            health: ModuleHealthStatus::default(),
1339            in_alternate_slot: false,
1340            draining_to_replace: false,
1341            configuration_updated_since_spawn: false,
1342        }
1343    }
1344}
1345
1346type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1347
1348type SpawnSubscriberKey = (ConnectionId, u64);
1349
1350#[derive(Debug)]
1351struct SpawnSubscriber {
1352    version: u8,
1353    frames: mpsc::Sender<Frame>,
1354    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1355    /// from which event. The full frame channel cannot carry that news, so it
1356    /// travels beside it; see `SpawnEventFeed::subscribe`.
1357    lagged: Option<oneshot::Sender<SpawnCursor>>,
1358}
1359
1360#[derive(Debug)]
1361struct SpawnEventState {
1362    daemon_incarnation: String,
1363    seq: u64,
1364    capacity: usize,
1365    live: HashMap<String, LiveSpawn>,
1366    generations: HashMap<String, u64>,
1367    events: VecDeque<SpawnEvent>,
1368    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1369}
1370
1371impl Default for SpawnEventState {
1372    fn default() -> Self {
1373        Self {
1374            daemon_incarnation: "unconfigured".to_string(),
1375            seq: 0,
1376            capacity: SPAWN_EVENT_RING_CAPACITY,
1377            live: HashMap::new(),
1378            generations: HashMap::new(),
1379            events: VecDeque::new(),
1380            subscribers: HashMap::new(),
1381        }
1382    }
1383}
1384
1385#[derive(Debug, Clone, Default)]
1386struct SpawnEventFeed(
1387    Arc<Mutex<SpawnEventState>>,
1388    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1389);
1390
1391#[derive(Debug, Clone, PartialEq, Eq)]
1392pub(crate) enum SpawnSubscribeRefusal {
1393    ForeignIncarnation { current: String },
1394    TooOld { oldest: SpawnCursor },
1395    Frame(String),
1396}
1397
1398impl SpawnEventFeed {
1399    fn configure_incarnation(&self, daemon_incarnation: String) {
1400        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1401        state.daemon_incarnation = daemon_incarnation;
1402        state.seq = 0;
1403        state.live.clear();
1404        state.generations.clear();
1405        state.events.clear();
1406        state.subscribers.clear();
1407    }
1408
1409    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1410        SpawnCursor {
1411            daemon_incarnation: state.daemon_incarnation.clone(),
1412            seq: state.seq,
1413        }
1414    }
1415
1416    fn snapshot(&self) -> SpawnSnapshot {
1417        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1418        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1419        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1420        SpawnSnapshot {
1421            cursor: Self::cursor(&state),
1422            ring_bound: state.capacity as u64,
1423            live,
1424        }
1425    }
1426
1427    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1428        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1429        let generation = state
1430            .generations
1431            .get(module_id)
1432            .copied()
1433            .unwrap_or(0)
1434            .checked_add(1)
1435            .expect("spawn generation exhausted");
1436        state.generations.insert(module_id.to_string(), generation);
1437        let live = LiveSpawn {
1438            module_id: module_id.to_string(),
1439            spawn_generation: generation,
1440            pid,
1441            spawned_at_ms,
1442        };
1443        state.live.insert(module_id.to_string(), live);
1444        Self::emit_locked(
1445            &mut state,
1446            SpawnEventKind::Spawned,
1447            module_id.to_string(),
1448            generation,
1449            pid,
1450            None,
1451            None,
1452        );
1453        generation
1454    }
1455
1456    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1457        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1458        let Some(live) = state.live.remove(module_id) else {
1459            warn!(
1460                module_id,
1461                "terminal record had no live spawn event identity"
1462            );
1463            return;
1464        };
1465        Self::emit_locked(
1466            &mut state,
1467            SpawnEventKind::Exited,
1468            module_id.to_string(),
1469            live.spawn_generation,
1470            live.pid,
1471            exit_code,
1472            exit_signal,
1473        );
1474    }
1475
1476    /// Report the exit of a process that a swap has already replaced.
1477    ///
1478    /// `emit_exited` removes the module's live entry, which after a swap's
1479    /// cutover describes the promoted candidate, not the old process now
1480    /// exiting. This emits the old generation's exit and leaves the live entry
1481    /// alone unless it still names that generation.
1482    fn emit_superseded_exited(
1483        &self,
1484        module_id: &str,
1485        spawn_generation: u64,
1486        pid: u32,
1487        exit_code: Option<i32>,
1488        exit_signal: Option<i32>,
1489    ) {
1490        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1491        if state
1492            .live
1493            .get(module_id)
1494            .is_some_and(|live| live.spawn_generation == spawn_generation)
1495        {
1496            state.live.remove(module_id);
1497        }
1498        Self::emit_locked(
1499            &mut state,
1500            SpawnEventKind::Exited,
1501            module_id.to_string(),
1502            spawn_generation,
1503            pid,
1504            exit_code,
1505            exit_signal,
1506        );
1507    }
1508
1509    #[allow(clippy::too_many_arguments)]
1510    fn emit_locked(
1511        state: &mut SpawnEventState,
1512        kind: SpawnEventKind,
1513        module_id: String,
1514        spawn_generation: u64,
1515        pid: u32,
1516        exit_code: Option<i32>,
1517        exit_signal: Option<i32>,
1518    ) {
1519        state.seq = state
1520            .seq
1521            .checked_add(1)
1522            .expect("spawn event sequence exhausted");
1523        let event = SpawnEvent {
1524            cursor: Self::cursor(state),
1525            kind,
1526            module_id,
1527            spawn_generation,
1528            pid,
1529            exit_code,
1530            exit_signal,
1531        };
1532        state.events.push_back(event.clone());
1533        while state.events.len() > state.capacity {
1534            state.events.pop_front();
1535        }
1536        let body = match serde_json::to_vec(&event) {
1537            Ok(body) => body,
1538            Err(error) => {
1539                error!(%error, "failed to serialize supervisor spawn event");
1540                return;
1541            }
1542        };
1543        state.subscribers.retain(|(connection_id, corr), subscriber| {
1544            let frame = Frame::build_with_version(
1545                subscriber.version,
1546                FrameType::StreamData,
1547                control_flags(),
1548                0,
1549                0,
1550                *corr,
1551                body.clone(),
1552            );
1553            match frame {
1554                Ok(frame) => {
1555                    if subscriber.frames.try_send(frame).is_ok() {
1556                        true
1557                    } else {
1558                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1559                        if let Some(lagged) = subscriber.lagged.take() {
1560                            let _ = lagged.send(event.cursor.clone());
1561                        }
1562                        false
1563                    }
1564                }
1565                Err(error) => {
1566                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1567                    false
1568                }
1569            }
1570        });
1571    }
1572
1573    fn subscribe(
1574        &self,
1575        connection_id: ConnectionId,
1576        corr: u64,
1577        version: u8,
1578        since: Option<SpawnCursor>,
1579        sink: FrameSink,
1580    ) -> Result<(), SpawnSubscribeRefusal> {
1581        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1582        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1583        {
1584            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1585            let replay = if let Some(since) = since {
1586                if since.daemon_incarnation != state.daemon_incarnation {
1587                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1588                        current: state.daemon_incarnation.clone(),
1589                    });
1590                }
1591                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1592                    if since.seq < oldest.seq.saturating_sub(1) {
1593                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1594                    }
1595                }
1596                state
1597                    .events
1598                    .iter()
1599                    .filter(|event| event.cursor.seq > since.seq)
1600                    .cloned()
1601                    .collect::<Vec<_>>()
1602            } else {
1603                Vec::new()
1604            };
1605            for event in replay {
1606                let body = serde_json::to_vec(&event)
1607                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1608                let frame = Frame::build_with_version(
1609                    version,
1610                    FrameType::StreamData,
1611                    control_flags(),
1612                    0,
1613                    0,
1614                    corr,
1615                    body,
1616                )
1617                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1618                frames
1619                    .try_send(frame)
1620                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1621            }
1622            state.subscribers.insert(
1623                (connection_id, corr),
1624                SpawnSubscriber {
1625                    version,
1626                    frames,
1627                    lagged: Some(lagged),
1628                },
1629            );
1630        }
1631        // The lagged terminal is sent here, by the forwarder, rather than by
1632        // the emitter: at the moment of the drop the subscriber's own channel
1633        // is full, and writing to the connection sink directly from the emitter
1634        // would put the Error AHEAD of the events still queued in that channel
1635        // (and the emitter holds the feed lock, so it cannot await the sink).
1636        // Dropping the subscriber drops the only sender, so `recv` drains every
1637        // queued event and then returns `None`; only then is the Error sent, so
1638        // the client sees each event it can keep, then the reason it was cut.
1639        // Cancel and connection removal drop the oneshot unsent, so they end
1640        // the stream with no Error.
1641        tokio::spawn(async move {
1642            while let Some(frame) = receiver.recv().await {
1643                if sink.send(frame).await.is_err() {
1644                    return;
1645                }
1646            }
1647            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1648                return;
1649            };
1650            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1651                Ok(frame) => {
1652                    let _ = sink.send(frame).await;
1653                }
1654                Err(error) => {
1655                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1656                }
1657            }
1658        });
1659        Ok(())
1660    }
1661
1662    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1663        let Some(subscriber) = self
1664            .0
1665            .lock()
1666            .unwrap_or_else(|p| p.into_inner())
1667            .subscribers
1668            .remove(&(connection_id, corr))
1669        else {
1670            return false;
1671        };
1672        if let Ok(frame) = Frame::build_with_version(
1673            subscriber.version,
1674            FrameType::StreamEnd,
1675            control_flags(),
1676            0,
1677            0,
1678            corr,
1679            Vec::new(),
1680        ) {
1681            tokio::spawn(async move {
1682                let _ = subscriber.frames.send(frame).await;
1683            });
1684        }
1685        true
1686    }
1687
1688    fn remove_connection(&self, connection_id: ConnectionId) {
1689        self.0
1690            .lock()
1691            .unwrap_or_else(|p| p.into_inner())
1692            .subscribers
1693            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1694    }
1695
1696    #[cfg(any(test, feature = "test-support"))]
1697    fn set_capacity(&self, capacity: usize) {
1698        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1699    }
1700
1701    #[cfg(any(test, feature = "test-support"))]
1702    fn subscriber_count(&self) -> usize {
1703        self.0
1704            .lock()
1705            .unwrap_or_else(|p| p.into_inner())
1706            .subscribers
1707            .len()
1708    }
1709}
1710
1711/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1712/// The terminal Error a lagged spawn subscriber receives after its queued events.
1713fn spawn_subscriber_lagged_frame(
1714    version: u8,
1715    corr: u64,
1716    first_undelivered: SpawnCursor,
1717) -> Result<Frame, String> {
1718    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1719        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1720        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1721            .to_string(),
1722        detail: Some(serde_json::json!({
1723            "first_undelivered_cursor": first_undelivered
1724        })),
1725    })
1726    .map_err(|error| error.to_string())?;
1727    Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1728        .map_err(|error| error.to_string())
1729}
1730
1731pub trait ModuleProcessLiveness: Send + Sync {
1732    fn process_live(&self, module_id: &str) -> Option<bool>;
1733
1734    /// Whether the supervisor is replacing this module's process right now: an
1735    /// operator restart or reload, a health restart, or a crash respawn whose
1736    /// backoff is running. A module in that state is not live, but a consumer
1737    /// refused now should retry shortly rather than treat the target as gone.
1738    /// Stopped, failed, and disabled modules are not replacing.
1739    fn process_replacing(&self, _module_id: &str) -> bool {
1740        false
1741    }
1742}
1743
1744/// Shared process-liveness registry keyed by supervised `module_id`.
1745#[derive(Debug, Clone, Default)]
1746pub struct SupervisorProcessLiveness {
1747    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1748}
1749
1750impl SupervisorProcessLiveness {
1751    pub fn new() -> Self {
1752        Self::default()
1753    }
1754
1755    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1756        let mut snapshots = self
1757            .snapshots
1758            .lock()
1759            .unwrap_or_else(|poisoned| poisoned.into_inner());
1760        snapshots.insert(module_id, snapshot);
1761    }
1762
1763    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1764        let mut snapshots = self
1765            .snapshots
1766            .lock()
1767            .unwrap_or_else(|poisoned| poisoned.into_inner());
1768        let is_current = snapshots
1769            .get(module_id)
1770            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1771            .unwrap_or(false);
1772        if is_current {
1773            snapshots.remove(module_id);
1774        }
1775    }
1776}
1777
1778impl ModuleProcessLiveness for SupervisorProcessLiveness {
1779    fn process_live(&self, module_id: &str) -> Option<bool> {
1780        let snapshot = {
1781            let snapshots = self
1782                .snapshots
1783                .lock()
1784                .unwrap_or_else(|poisoned| poisoned.into_inner());
1785            snapshots.get(module_id).cloned()
1786        }?;
1787        let snapshot = snapshot
1788            .lock()
1789            .unwrap_or_else(|poisoned| poisoned.into_inner());
1790        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1791    }
1792
1793    fn process_replacing(&self, module_id: &str) -> bool {
1794        let Some(snapshot) = self
1795            .snapshots
1796            .lock()
1797            .unwrap_or_else(|poisoned| poisoned.into_inner())
1798            .get(module_id)
1799            .cloned()
1800        else {
1801            return false;
1802        };
1803        let snapshot = snapshot
1804            .lock()
1805            .unwrap_or_else(|poisoned| poisoned.into_inner());
1806        snapshot.enabled
1807            && match snapshot.state {
1808                ModuleState::Restarting => true,
1809                ModuleState::Draining => snapshot.draining_to_replace,
1810                ModuleState::Starting
1811                | ModuleState::Running
1812                | ModuleState::Unresponsive
1813                | ModuleState::Stopped
1814                | ModuleState::Failed
1815                | ModuleState::Disabled => false,
1816            }
1817    }
1818}
1819
1820#[cfg(test)]
1821#[derive(Debug, Default)]
1822struct ReloadExitRecordGate {
1823    reached: tokio::sync::Notify,
1824    resume: tokio::sync::Notify,
1825}
1826
1827#[derive(Debug, Clone, Copy)]
1828enum RespawnKind {
1829    Spawn,
1830    Reload,
1831}
1832
1833#[derive(Debug, Clone, Copy)]
1834struct PendingRespawn {
1835    deadline: Instant,
1836    kind: RespawnKind,
1837}
1838
1839type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1840
1841#[derive(Debug, Clone)]
1842struct SupervisorRuntimeConfig {
1843    start_requester: Option<Principal>,
1844    pending_activity_restart: bool,
1845    provenance_probe: ExecutableIdentityProbe,
1846    /// Restart operations hand their backoff to the loop, which keeps serving commands.
1847    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1848    /// A reload acknowledges completion only after its replacement registers.
1849    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1850    restart_policy: RestartPolicy,
1851    /// This module's RESOLVED drain budget: per-module config when present,
1852    /// else `default_drain_timeout`.
1853    drain_timeout: Duration,
1854    /// Shared with the status handle so the attested value changes atomically
1855    /// when a rescan updates the running drain policy.
1856    effective_drain_timeout: Arc<Mutex<Duration>>,
1857    /// The supervisor-wide fallback, kept so a configuration update that
1858    /// REMOVES the per-module override can re-resolve to it.
1859    default_drain_timeout: Duration,
1860    health: HealthConfig,
1861    connection_file_path: Option<PathBuf>,
1862    capture_logs_dir: Option<PathBuf>,
1863    forwarding: Option<Arc<ForwardingTable>>,
1864    /// The shared handle, so every spawn path (initial, restart, reload) records the
1865    /// reserved-module launch nonce the HELLO verifier checks against.
1866    supervisor_handle: Option<SupervisorHandle>,
1867    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
1868    /// status queries.
1869    ///
1870    /// One ring per module, held across every respawn. The lines explaining an exit
1871    /// are written BEFORE that exit, so a ring recreated per process would be empty
1872    /// exactly when it is asked for.
1873    stderr_ring: Arc<Mutex<StderrRing>>,
1874    terminal_ring: Arc<Mutex<TerminalRing>>,
1875    spawn_events: SpawnEventFeed,
1876    child_roster: ChildRoster,
1877    #[cfg(target_os = "linux")]
1878    cgroup_placement: Option<subc_cgroup::Placement>,
1879    #[cfg(test)]
1880    test_seed_stale_facts_before_enable_spawn: bool,
1881    #[cfg(test)]
1882    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1883}
1884
1885#[derive(Debug, Clone, PartialEq, Eq)]
1886struct SupervisedConfiguration {
1887    spec: ModuleSpec,
1888    health: HealthConfig,
1889}
1890
1891/// Shared daemon lookup table for supervised module handles.
1892///
1893/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
1894/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
1895/// launch nonces recorded at spawn are checked by the same daemon instance.
1896#[derive(Debug, Clone, Default)]
1897pub struct SupervisorHandle {
1898    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1899    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1900    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1901    /// Module ids the supervisor has taken on. An id is added BEFORE the
1902    /// module's first process is spawned and removed only when the module
1903    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
1904    /// the keys of `modules`.
1905    ///
1906    /// `modules` cannot answer "is this module configured?" on its own: a
1907    /// [`SupervisedModule`] only exists once its process has been spawned, and
1908    /// a fast child can connect, register, sync its scopes and ask about them
1909    /// before the supervisor has inserted it. Answering "not configured" in that
1910    /// gap makes scope admission refuse with the terminal "will never sync"
1911    /// instead of the retryable "has not synced yet".
1912    configured_ids: Arc<Mutex<HashSet<String>>>,
1913    spawn_events: SpawnEventFeed,
1914    /// The current expected launch nonce for each reserved module_id. Set when the
1915    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
1916    /// non-reserved module never has an entry here and is never nonce-checked.
1917    /// Reserved module ids and the nonce that authorizes their next HELLO.
1918    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
1919    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
1920    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
1921    /// had NO entry and admitted anyone: the reservation protected the nonce
1922    /// holder, not the NAME (found live by CKCRED's canary probe registering
1923    /// against a reserved scratch id).
1924    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
1925    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
1926    ///
1927    /// This is deliberately in-memory only: subc is state-free across daemon
1928    /// restarts, and the tombstone only explains the hours-after-removal window
1929    /// while this executing daemon is still alive. Do not persist it in a store.
1930    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
1931    /// The current launch nonce for every supervised spawn. This is separate from
1932    /// reserved_nonces because consumer route.open attestation applies to all spawned
1933    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
1934    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
1935    /// Reserved namespace prefixes mapped to the supervised owner module whose
1936    /// current spawn nonce authorizes HELLO claims below the prefix.
1937    ///
1938    /// Per §2.6 this is not a same-user security barrier: a same-user process can
1939    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
1940    /// accidental collisions and lower-trust processes from squatting protected
1941    /// namespaces.
1942    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
1943    /// Blue/green swaps in progress, by module id. An entry exists from just
1944    /// before the candidate process is spawned until the swap has failed, or
1945    /// has cut over and the old process is gone. While it exists, HELLO for the
1946    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
1947    /// consumer attestation accepts both processes' nonces.
1948    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
1949    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
1950    promotion_observer: PromotionObserverSlot,
1951    /// Serializes module-set reconciliation with operator lifecycle commands. Without
1952    /// this daemon-wide ordering, a rescan could retire or update a module while a
1953    /// concurrent reload still held its old handle and launch specification.
1954    operation_lock: Arc<AsyncMutex<()>>,
1955}
1956
1957/// Told when a swap has promoted its candidate to be the module's active
1958/// registration.
1959///
1960/// An ordinary HELLO runs the control plane's registration side effects (the
1961/// capability cache, the deny census, the requirement recompute) as it
1962/// registers. A swap candidate's HELLO does not, because it is not routable;
1963/// promotion is when those must run instead, and promotion happens in the
1964/// supervisor, which has no other way into the control handler.
1965pub(crate) trait SwapPromotionObserver: Send + Sync {
1966    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
1967}
1968
1969/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
1970/// control handler) owns this handle, so a strong reference back would be a
1971/// cycle that keeps both alive.
1972#[derive(Clone, Default)]
1973struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
1974
1975impl fmt::Debug for PromotionObserverSlot {
1976    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1977        f.write_str("PromotionObserverSlot")
1978    }
1979}
1980
1981/// The nonces of one open swap.
1982#[derive(Debug, Clone)]
1983struct OpenSwap {
1984    /// The launch nonce minted for the candidate process. It is the swap
1985    /// token: the only thing that admits a HELLO into the candidate slot.
1986    candidate_nonce: String,
1987    /// The incumbent's launch nonce, captured when the swap opened. It is kept
1988    /// here because cutover moves the module's recorded spawn nonce to the
1989    /// candidate while the incumbent is still draining and its consumers are
1990    /// still attesting with this one.
1991    incumbent_nonce: Option<String>,
1992    /// Set once a HELLO has been admitted with the swap token, so the token
1993    /// admits one registration and cannot be replayed after cutover empties
1994    /// the candidate slot.
1995    candidate_admitted: bool,
1996}
1997
1998/// What the swap gate says about a HELLO. See
1999/// [`SupervisorHandle::swap_hello_admission`].
2000#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2001pub(crate) enum SwapHelloAdmission {
2002    /// No swap is open for the id (or the HELLO carries the incumbent's own
2003    /// nonce); the ordinary gates decide.
2004    NotSwapping,
2005    /// The HELLO carries the swap token: register it into the candidate slot.
2006    Candidate,
2007    /// A swap is open and the HELLO carries a nonce the supervisor did not
2008    /// mint for this id, no nonce, or a token already used.
2009    Refused,
2010}
2011
2012#[derive(Debug, Clone, PartialEq, Eq)]
2013pub(crate) enum ReservedHelloRejection {
2014    Exact {
2015        module_id: String,
2016    },
2017    Prefix {
2018        prefix: String,
2019        owner_module_id: String,
2020    },
2021}
2022
2023impl SupervisorHandle {
2024    pub fn new() -> Self {
2025        Self::default()
2026    }
2027
2028    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2029        self.spawn_events.snapshot()
2030    }
2031
2032    pub(crate) fn subscribe_spawns(
2033        &self,
2034        connection_id: ConnectionId,
2035        corr: u64,
2036        version: u8,
2037        since: Option<SpawnCursor>,
2038        sink: FrameSink,
2039    ) -> Result<(), SpawnSubscribeRefusal> {
2040        self.spawn_events
2041            .subscribe(connection_id, corr, version, since, sink)
2042    }
2043
2044    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2045        self.spawn_events.cancel(connection_id, corr)
2046    }
2047
2048    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2049        self.spawn_events.remove_connection(connection_id);
2050    }
2051
2052    #[cfg(any(test, feature = "test-support"))]
2053    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2054        assert!(capacity > 0, "spawn event capacity must be non-zero");
2055        self.spawn_events.set_capacity(capacity);
2056    }
2057
2058    #[cfg(any(test, feature = "test-support"))]
2059    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2060        self.spawn_events.subscriber_count()
2061    }
2062
2063    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2064    /// a respawn invalidates stale consumer identities.
2065    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2066        self.spawn_nonces
2067            .lock()
2068            .unwrap_or_else(|poisoned| poisoned.into_inner())
2069            .insert(module_id.to_string(), nonce);
2070    }
2071
2072    /// Record the launch nonce expected from the next HELLO for a reserved module,
2073    /// replacing any prior nonce (a respawn invalidates the previous one).
2074    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2075        self.reserved_nonces
2076            .lock()
2077            .unwrap_or_else(|poisoned| poisoned.into_inner())
2078            .insert(module_id.to_string(), Some(nonce));
2079    }
2080
2081    /// Record namespace prefixes owned by a supervised module.
2082    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2083        let mut owners = self
2084            .reserved_prefix_owners
2085            .lock()
2086            .unwrap_or_else(|poisoned| poisoned.into_inner());
2087        owners.retain(|_, owner| owner != owner_module_id);
2088        for prefix in prefixes {
2089            owners.insert(prefix.clone(), owner_module_id.to_string());
2090        }
2091    }
2092
2093    /// The launch nonce most recently minted for a module's spawn, if any.
2094    #[cfg(test)]
2095    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2096        self.spawn_nonces
2097            .lock()
2098            .unwrap_or_else(|poisoned| poisoned.into_inner())
2099            .get(module_id)
2100            .cloned()
2101    }
2102
2103    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2104        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2105        let spawn_nonce = self
2106            .spawn_nonces
2107            .lock()
2108            .unwrap_or_else(|poisoned| poisoned.into_inner())
2109            .get(&spec.module_id)
2110            .cloned();
2111        let mut reserved_nonces = self
2112            .reserved_nonces
2113            .lock()
2114            .unwrap_or_else(|poisoned| poisoned.into_inner());
2115        if spec.reserved {
2116            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2117            // reserved name whose module has never spawned has no legitimate
2118            // holder, and the entry's absence is what used to leave the name
2119            // open to the first claimant.
2120            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2121        }
2122        drop(reserved_nonces);
2123        // A later unreserved declaration must not silently unreserve an id that
2124        // was retained after its reserved configuration was removed. The explicit
2125        // release ceremony is the only operation that retires that gate.
2126        self.removal_tombstones
2127            .lock()
2128            .unwrap_or_else(|poisoned| poisoned.into_inner())
2129            .remove(&spec.module_id);
2130    }
2131
2132    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2133    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2134    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2135    /// with no matching prefix are always authorized.
2136    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2137        self.reserved_hello_rejection(module_id, presented)
2138            .is_none()
2139    }
2140
2141    pub(crate) fn reserved_hello_rejection(
2142        &self,
2143        module_id: &str,
2144        presented: Option<&str>,
2145    ) -> Option<ReservedHelloRejection> {
2146        let nonces = self
2147            .reserved_nonces
2148            .lock()
2149            .unwrap_or_else(|poisoned| poisoned.into_inner());
2150        if let Some(expected) = nonces.get(module_id) {
2151            // `None` = reserved with no legitimate holder: refuse every
2152            // presentation, because no process can hold a nonce that was never
2153            // minted. Only a real minted nonce admits, in constant time.
2154            let authorized = match expected {
2155                Some(expected) => {
2156                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2157                }
2158                None => false,
2159            };
2160            if authorized {
2161                return None;
2162            }
2163            return Some(ReservedHelloRejection::Exact {
2164                module_id: module_id.to_string(),
2165            });
2166        }
2167        drop(nonces);
2168
2169        let matched_prefix = self
2170            .reserved_prefix_owners
2171            .lock()
2172            .unwrap_or_else(|poisoned| poisoned.into_inner())
2173            .iter()
2174            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2175            .max_by_key(|(prefix, _)| prefix.len())
2176            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2177        let (prefix, owner_module_id) = matched_prefix?;
2178
2179        let authorized = presented.is_some_and(|presented| {
2180            self.spawn_nonces
2181                .lock()
2182                .unwrap_or_else(|poisoned| poisoned.into_inner())
2183                .get(&owner_module_id)
2184                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2185                // While the owner is being swapped, children started by
2186                // either of its two processes hold that process's nonce.
2187                || self.swap_nonce_matches(&owner_module_id, presented)
2188        });
2189        if authorized {
2190            None
2191        } else {
2192            Some(ReservedHelloRejection::Prefix {
2193                prefix,
2194                owner_module_id,
2195            })
2196        }
2197    }
2198
2199    /// Whether a consumer connection proved it came from a daemon-spawned module.
2200    ///
2201    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2202    /// accepted only for module ids the supervisor has spawned.
2203    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2204        if presented.is_empty() {
2205            return false;
2206        }
2207        let nonces = self
2208            .spawn_nonces
2209            .lock()
2210            .unwrap_or_else(|poisoned| poisoned.into_inner());
2211        let current = nonces
2212            .get(module_id)
2213            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2214        drop(nonces);
2215        // During a swap two processes of the module are alive, and a consumer
2216        // started by either one presents that process's nonce. Accepting only
2217        // the recorded one would fail the incumbent's consumers for the whole
2218        // overlap once cutover moves the record to the candidate.
2219        current || self.swap_nonce_matches(module_id, presented)
2220    }
2221
2222    /// Whether `presented` is either nonce of an open swap for `module_id`.
2223    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2224        let swaps = self
2225            .swaps
2226            .lock()
2227            .unwrap_or_else(|poisoned| poisoned.into_inner());
2228        swaps.get(module_id).is_some_and(|swap| {
2229            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2230                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2231                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2232                })
2233        })
2234    }
2235
2236    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2237    /// Called before the candidate process exists.
2238    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2239        let incumbent_nonce = self
2240            .spawn_nonces
2241            .lock()
2242            .unwrap_or_else(|poisoned| poisoned.into_inner())
2243            .get(module_id)
2244            .cloned();
2245        self.swaps
2246            .lock()
2247            .unwrap_or_else(|poisoned| poisoned.into_inner())
2248            .insert(
2249                module_id.to_string(),
2250                OpenSwap {
2251                    candidate_nonce,
2252                    incumbent_nonce,
2253                    candidate_admitted: false,
2254                },
2255            );
2256    }
2257
2258    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2259    /// the module's recorded one.
2260    pub(crate) fn close_swap(&self, module_id: &str) {
2261        self.swaps
2262            .lock()
2263            .unwrap_or_else(|poisoned| poisoned.into_inner())
2264            .remove(module_id);
2265    }
2266
2267    /// Install the observer told about swap promotions, replacing any earlier
2268    /// one.
2269    pub(crate) fn set_swap_promotion_observer(
2270        &self,
2271        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2272    ) {
2273        *self
2274            .promotion_observer
2275            .0
2276            .lock()
2277            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2278    }
2279
2280    /// Tell the installed observer, if it is still alive, that a swap promoted
2281    /// `registration`.
2282    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2283        let observer = self
2284            .promotion_observer
2285            .0
2286            .lock()
2287            .unwrap_or_else(|poisoned| poisoned.into_inner())
2288            .as_ref()
2289            .and_then(std::sync::Weak::upgrade);
2290        if let Some(observer) = observer {
2291            observer.swap_promoted(registration);
2292        }
2293    }
2294
2295    /// Whether a swap is open for `module_id`.
2296    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2297        self.swaps
2298            .lock()
2299            .unwrap_or_else(|poisoned| poisoned.into_inner())
2300            .contains_key(module_id)
2301    }
2302
2303    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2304    /// respawn would, once cutover has made the candidate the module's process.
2305    /// The swap stays open so the incumbent's nonce keeps attesting until the
2306    /// incumbent has drained and exited.
2307    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2308        let candidate_nonce = self
2309            .swaps
2310            .lock()
2311            .unwrap_or_else(|poisoned| poisoned.into_inner())
2312            .get(module_id)
2313            .map(|swap| swap.candidate_nonce.clone());
2314        let Some(nonce) = candidate_nonce else {
2315            return;
2316        };
2317        self.set_spawn_nonce(module_id, nonce.clone());
2318        if reserved {
2319            self.set_reserved_nonce(module_id, nonce);
2320        }
2321    }
2322
2323    /// The swap gate for a HELLO claiming `module_id`.
2324    ///
2325    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2326    /// presents the candidate nonce, which the reserved gate (holding the
2327    /// incumbent's nonce) would refuse as `reserved_module` before swap
2328    /// admission was ever reached. And it applies to unreserved ids too: for an
2329    /// unreserved id the only thing that ever stopped a second process claiming
2330    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2331    /// refusal a swap lifts for its candidate.
2332    ///
2333    /// The incumbent's own nonce falls through to the ordinary gates, which
2334    /// treat it as they always have (a live incumbent is refused as a
2335    /// duplicate). Anything else while a swap is open is refused, including an
2336    /// absent nonce.
2337    pub(crate) fn swap_hello_admission(
2338        &self,
2339        module_id: &str,
2340        presented: Option<&str>,
2341    ) -> SwapHelloAdmission {
2342        let swaps = self
2343            .swaps
2344            .lock()
2345            .unwrap_or_else(|poisoned| poisoned.into_inner());
2346        let Some(swap) = swaps.get(module_id) else {
2347            return SwapHelloAdmission::NotSwapping;
2348        };
2349        let Some(presented) = presented else {
2350            return SwapHelloAdmission::Refused;
2351        };
2352        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2353            return if swap.candidate_admitted {
2354                SwapHelloAdmission::Refused
2355            } else {
2356                SwapHelloAdmission::Candidate
2357            };
2358        }
2359        if swap
2360            .incumbent_nonce
2361            .as_deref()
2362            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2363        {
2364            return SwapHelloAdmission::NotSwapping;
2365        }
2366        SwapHelloAdmission::Refused
2367    }
2368
2369    /// Record that the swap token has registered a candidate, so it admits no
2370    /// second HELLO.
2371    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2372        if let Some(swap) = self
2373            .swaps
2374            .lock()
2375            .unwrap_or_else(|poisoned| poisoned.into_inner())
2376            .get_mut(module_id)
2377        {
2378            swap.candidate_admitted = true;
2379        }
2380    }
2381
2382    /// Test/support lookup for the current launch nonce of a supervised spawn.
2383    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2384        self.spawn_nonces
2385            .lock()
2386            .unwrap_or_else(|poisoned| poisoned.into_inner())
2387            .get(module_id)
2388            .cloned()
2389    }
2390
2391    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2392    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2393        self.reserved_nonces
2394            .lock()
2395            .unwrap_or_else(|poisoned| poisoned.into_inner())
2396            .get(module_id)
2397            .cloned()
2398            .flatten()
2399    }
2400
2401    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2402        // Normally already marked before the process was spawned; marking here
2403        // too keeps `configured_ids` a superset of the roster for any caller
2404        // that inserts a module directly.
2405        self.mark_configured(module.module_id());
2406        let mut modules = self
2407            .modules
2408            .lock()
2409            .unwrap_or_else(|poisoned| poisoned.into_inner());
2410        modules.insert(module.module_id().to_string(), module)
2411    }
2412
2413    /// Record that the supervisor has taken on `module_id`. Called before the
2414    /// module's first process is spawned, so that by the time that process can
2415    /// register, [`Self::is_configured`] already answers true.
2416    fn mark_configured(&self, module_id: &str) {
2417        self.configured_ids
2418            .lock()
2419            .unwrap_or_else(|poisoned| poisoned.into_inner())
2420            .insert(module_id.to_string());
2421    }
2422
2423    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2424    /// before it was ever put on the roster. A module already on the roster
2425    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2426    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2427        let modules = self
2428            .modules
2429            .lock()
2430            .unwrap_or_else(|poisoned| poisoned.into_inner());
2431        if !modules.contains_key(module_id) {
2432            self.configured_ids
2433                .lock()
2434                .unwrap_or_else(|poisoned| poisoned.into_inner())
2435                .remove(module_id);
2436        }
2437    }
2438
2439    /// Whether `module_id` is a module this daemon supervises: on the roster,
2440    /// or about to be (its process is being spawned right now).
2441    ///
2442    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2443    /// for scopes: a supervised module's process can register and sync before
2444    /// [`Self::get`] can return it, and in that window it is still a module
2445    /// that will sync, not one that never will.
2446    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2447        self.configured_ids
2448            .lock()
2449            .unwrap_or_else(|poisoned| poisoned.into_inner())
2450            .contains(module_id)
2451    }
2452
2453    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2454        let modules = self
2455            .modules
2456            .lock()
2457            .unwrap_or_else(|poisoned| poisoned.into_inner());
2458        modules.get(module_id).cloned()
2459    }
2460
2461    pub(crate) fn record_late_health_answer(
2462        &self,
2463        module_id: &str,
2464        latency_ms: u64,
2465    ) -> Result<bool, SuperviseError> {
2466        let Some(module) = self.get(module_id) else {
2467            return Ok(false);
2468        };
2469        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2470            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2471            state.health.last_late_answer_latency_ms = Some(latency_ms);
2472            // A late answer is an answer: the module served the probe, just past
2473            // the deadline. Leaving the miss streak in place while logging
2474            // "proves the module is alive" is how a CPU-starved module that
2475            // answers every probe a few seconds late still marches to the
2476            // threshold and gets killed — the exact kill class `NoAnswer` is
2477            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2478            // is degradation, and degradation reports; it does not restart.
2479            state.health.consecutive_failures = 0;
2480        })?;
2481        Ok(true)
2482    }
2483
2484    /// Arm the one-shot marker for the module process that this caller
2485    /// deliberately initiated severance against. Generic connection teardown
2486    /// must not call this:
2487    /// a surviving process would otherwise retain an exemption for a later
2488    /// genuine crash.
2489    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2490        let Some(module) = self.get(module_id) else {
2491            return Ok(false);
2492        };
2493        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2494        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2495            return Ok(false);
2496        };
2497        drop(snapshot);
2498        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2499    }
2500
2501    pub fn list(&self) -> Vec<SupervisedModule> {
2502        let modules = self
2503            .modules
2504            .lock()
2505            .unwrap_or_else(|poisoned| poisoned.into_inner());
2506        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2507        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2508        modules
2509    }
2510
2511    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2512        self.activity_images
2513            .lock()
2514            .unwrap_or_else(|p| p.into_inner())
2515            .remove(module_id);
2516        self.spawn_nonces
2517            .lock()
2518            .unwrap_or_else(|poisoned| poisoned.into_inner())
2519            .remove(module_id);
2520        self.close_swap(module_id);
2521        let mut reserved_nonces = self
2522            .reserved_nonces
2523            .lock()
2524            .unwrap_or_else(|poisoned| poisoned.into_inner());
2525        if reserved_nonces.contains_key(module_id) {
2526            // The old nonce must die with the removed process, but the exact-id
2527            // gate remains until an operator explicitly releases it.
2528            reserved_nonces.insert(module_id.to_string(), None);
2529        }
2530        drop(reserved_nonces);
2531        self.reserved_prefix_owners
2532            .lock()
2533            .unwrap_or_else(|poisoned| poisoned.into_inner())
2534            .retain(|_, owner| owner != module_id);
2535        let removed = self
2536            .modules
2537            .lock()
2538            .unwrap_or_else(|poisoned| poisoned.into_inner())
2539            .remove(module_id);
2540        self.configured_ids
2541            .lock()
2542            .unwrap_or_else(|poisoned| poisoned.into_inner())
2543            .remove(module_id);
2544        removed
2545    }
2546
2547    /// Remember a module removed by a non-preview rescan so route.open can
2548    /// distinguish that intentional removal from an unknown id.
2549    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2550        self.removal_tombstones
2551            .lock()
2552            .unwrap_or_else(|poisoned| poisoned.into_inner())
2553            .insert(module_id.to_string(), unix_ms_now());
2554    }
2555
2556    /// Return how long ago a rescan removed this module in milliseconds.
2557    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2558        self.removal_tombstones
2559            .lock()
2560            .unwrap_or_else(|poisoned| poisoned.into_inner())
2561            .get(module_id)
2562            .copied()
2563            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2564    }
2565
2566    /// Retire a reserved-id gate only after its module has left supervision.
2567    ///
2568    /// A retained gate has no live nonce (`None`), so releasing any other entry
2569    /// would weaken a currently configured or otherwise active reservation.
2570    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2571        if self.get(module_id).is_some() {
2572            return false;
2573        }
2574        let mut reserved_nonces = self
2575            .reserved_nonces
2576            .lock()
2577            .unwrap_or_else(|poisoned| poisoned.into_inner());
2578        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2579            return false;
2580        }
2581        reserved_nonces.remove(module_id);
2582        true
2583    }
2584
2585    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2586        Arc::clone(&self.operation_lock)
2587    }
2588}
2589
2590/// Process supervisor for subc-owned singleton modules.
2591#[derive(Debug, Clone)]
2592pub struct Supervisor {
2593    registry: Arc<Registry>,
2594    restart_policy: RestartPolicy,
2595    drain_timeout: Duration,
2596    connection_file_path: Option<PathBuf>,
2597    capture_logs_dir: Option<PathBuf>,
2598    forwarding: Option<Arc<ForwardingTable>>,
2599    process_liveness: Arc<SupervisorProcessLiveness>,
2600    supervisor_handle: Option<SupervisorHandle>,
2601    health: HealthConfig,
2602    daemon_start_clock: crate::clock::StartClock,
2603    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2604    spawn_events: SpawnEventFeed,
2605    provenance_probe: ExecutableIdentityProbe,
2606    /// Every process spawned through this supervisor (and its clones) and not
2607    /// yet reaped, so daemon shutdown can end them.
2608    child_roster: ChildRoster,
2609    #[cfg(target_os = "linux")]
2610    cgroup_placement: Option<subc_cgroup::Placement>,
2611    #[cfg(test)]
2612    test_after_first_spawn: AfterFirstSpawnHook,
2613}
2614
2615/// A roster entry whose monitor and first process have not started yet.
2616/// Keeping the command receiver here lets discovery serve status immediately
2617/// while start/stop requests wait until bootstrap has published the connection file.
2618pub(crate) struct RegisteredConfiguredModule {
2619    spec: ModuleSpec,
2620    runtime: SupervisorRuntimeConfig,
2621    module: SupervisedModule,
2622    commands: mpsc::Receiver<SupervisorCommand>,
2623}
2624
2625/// Test-only hook run on the path that takes on a new module, right after its
2626/// first `spawn_child` returns (the process exists and could already be
2627/// registering) and before that process is handed to the module's supervise
2628/// loop and put on the roster. Lets a test observe what a fast child would see
2629/// in that window without racing a real one.
2630#[cfg(test)]
2631#[derive(Clone, Default)]
2632struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2633
2634#[cfg(test)]
2635type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2636
2637#[cfg(test)]
2638impl fmt::Debug for AfterFirstSpawnHook {
2639    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2640        f.write_str("AfterFirstSpawnHook")
2641    }
2642}
2643
2644#[cfg(test)]
2645impl AfterFirstSpawnHook {
2646    fn run(&self, module_id: &str) {
2647        if let Some(hook) = &self.0 {
2648            hook(module_id);
2649        }
2650    }
2651}
2652
2653impl Supervisor {
2654    #[cfg(test)]
2655    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2656        let supervisor = Self::new(registry, policy);
2657        #[cfg(target_os = "macos")]
2658        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2659        supervisor
2660    }
2661    /// Verify the trampoline once when configured. Missing private OS support
2662    /// refuses every macOS launch by name but does not stop the daemon's control
2663    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2664    /// entry point; the library must not exec an arbitrary hosting program.
2665    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2666        let path = path.into();
2667        #[cfg(target_os = "macos")]
2668        {
2669            let result = probe_privacy_trampoline(&path).map(|()| path);
2670            if let Err(cause) = &result {
2671                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2672            }
2673            self.child_roster.set_privacy_trampoline(result);
2674        }
2675        #[cfg(not(target_os = "macos"))]
2676        let _ = path;
2677        self
2678    }
2679    /// The first step of an announced daemon shutdown, before the notice and
2680    /// before any connection is closed.
2681    ///
2682    /// Sets the daemon-shutdown flag first: from here on no module is
2683    /// respawned (crash restart, operator restart, or swap), and every child
2684    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2685    /// the module exits on the EOF this shutdown gives it or is signalled by a
2686    /// service manager that kills the whole cgroup. Then writes the journal's
2687    /// shutdown marker, which records the instant and closes this daemon
2688    /// incarnation's stretch of the journal.
2689    #[cfg(unix)]
2690    pub(crate) fn begin_daemon_shutdown(&self) {
2691        self.child_roster.close();
2692        if let Some(journal) = &self.terminal_journal {
2693            journal.stamp_shutdown();
2694        }
2695    }
2696
2697    /// Announce a cut while established connections can still carry replies.
2698    /// These budgets promise notice and a bounded wait, not child completion;
2699    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2700    #[cfg(unix)]
2701    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2702        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2703        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2704        let Some(forwarding) = &self.forwarding else {
2705            return Ok(());
2706        };
2707        let modules = forwarding
2708            .begin_daemon_drain()
2709            .map_err(SuperviseError::Forwarding)?;
2710        let deadline_ms =
2711            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2712        let mut notices = tokio::task::JoinSet::new();
2713        let mut drains = Vec::new();
2714        for (module_id, endpoint) in modules {
2715            let Some(target) = forwarding
2716                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2717                .map_err(SuperviseError::Forwarding)?
2718            else {
2719                continue;
2720            };
2721            let routes = forwarding
2722                .endpoint_routes(target.endpoint)
2723                .map_err(SuperviseError::Forwarding)?;
2724            // Restart allows deployed consumers to reopen after the new daemon
2725            // appears. The wire reason stays `restart`; what tells a daemon cut
2726            // apart from a module restart afterwards is the terminal record
2727            // itself, whose disposition is `daemon_shutdown` for every exit
2728            // observed once `begin_daemon_shutdown` has run.
2729            let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2730                reason: RouteCloseReason::Restart,
2731                deadline_ms,
2732            })
2733            .expect("module draining serializes");
2734            let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2735            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2736            for route in routes {
2737                let client = route.goodbye_target;
2738                if let Some((_, channels)) = clients
2739                    .iter_mut()
2740                    .find(|(existing, _)| existing.connection_id == client.connection_id)
2741                {
2742                    channels.push(client.channel);
2743                } else {
2744                    let channel = client.channel;
2745                    clients.push((client, vec![channel]));
2746                }
2747            }
2748            for (client, mut channels) in clients {
2749                channels.sort_unstable();
2750                channels.dedup();
2751                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2752                    module_id: module_id.clone(),
2753                    channels,
2754                    reason: RouteCloseReason::Restart,
2755                })
2756                .expect("route closing serializes");
2757                recipients.push((client.sink, client.negotiated_ver, closing));
2758            }
2759            for (sink, version, body) in recipients {
2760                notices.spawn(async move {
2761                    let frame = Frame::build_with_version(
2762                        version,
2763                        FrameType::Push,
2764                        control_flags(),
2765                        0,
2766                        0,
2767                        0,
2768                        body,
2769                    )
2770                    .expect("bounded lifecycle notice frame builds");
2771                    sink.send_flushed(frame).await
2772                });
2773            }
2774            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2775            drains.push((module_id, target.endpoint, gauges));
2776        }
2777        // A quiet forwarding table is not proof that queued notices reached the
2778        // socket. Wait for writer flush acknowledgements before testing quiescence.
2779        let notice_deadline = Instant::now() + NOTICE_BUDGET;
2780        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2781            if !matches!(result, Ok(Ok(()))) {
2782                warn!(?result, "daemon shutdown notice delivery failed");
2783            }
2784        }
2785        notices.abort_all();
2786        let deadline = Instant::now() + DRAIN_BUDGET;
2787        let mut waits = tokio::task::JoinSet::new();
2788        for (module_id, endpoint, gauges) in drains {
2789            let forwarding = Arc::clone(forwarding);
2790            let mut runtime = self.runtime_config();
2791            runtime.health.cadence = Duration::from_millis(100);
2792            waits.spawn(async move {
2793                let wait_result = wait_for_forwarding_quiescence(
2794                    &forwarding,
2795                    &module_id,
2796                    &runtime,
2797                    endpoint,
2798                    deadline,
2799                    &gauges,
2800                    DrainScope::Endpoint(endpoint),
2801                )
2802                .await;
2803                // Store the measurement before the task returns: quiescence
2804                // may be reached just as the deadline stops the join loop from
2805                // collecting results. Unfinished or failed waits report false.
2806                forwarding
2807                    .record_daemon_drain_result(
2808                        endpoint,
2809                        drained_after_quiescence_wait(&wait_result),
2810                    )
2811                    .map_err(SuperviseError::Forwarding)?;
2812                wait_result
2813            });
2814        }
2815        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2816            if !matches!(result, Ok(Ok(true))) {
2817                warn!(?result, "daemon shutdown drain did not reach quiescence");
2818            }
2819        }
2820        Ok(())
2821    }
2822
2823    /// The last step of an announced daemon shutdown, after the notice and the
2824    /// drain: send every registered module a module GOODBYE, the same planned
2825    /// stop signal `ck module stop` gives, then close every connection so each
2826    /// subc module sees EOF and starts its own teardown, then end every
2827    /// supervised child that has not exited
2828    /// by its own deadline (its drain budget, capped). Modules lead their own
2829    /// process groups, so a
2830    /// service manager's group kill no longer reaches them; without this a
2831    /// child that does not stop on EOF (every `protocol: "none"` child, which
2832    /// has no connection) would outlive the daemon. Every wait is bounded (see
2833    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
2834    #[cfg(unix)]
2835    pub(crate) async fn end_children_for_daemon_shutdown(
2836        &self,
2837        already_escalated: bool,
2838        escalate: impl std::future::Future<Output = ()>,
2839    ) {
2840        tokio::pin!(escalate);
2841        let mut escalated = already_escalated;
2842        if let Some(forwarding) = &self.forwarding {
2843            let reason = CloseReason::new(
2844                "daemon_shutdown",
2845                "the daemon is exiting after its shutdown notice and drain",
2846            );
2847            if escalated {
2848                // The operator asked to stop waiting: queue the GOODBYEs but
2849                // do not wait for them to be written.
2850                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2851            } else {
2852                tokio::select! {
2853                    biased;
2854                    _ = escalate.as_mut() => {
2855                        info!("second SIGTERM: abandoning module GOODBYE delivery");
2856                        escalated = true;
2857                    }
2858                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2859                }
2860            }
2861            let closed = forwarding.close_all_connections(&reason);
2862            debug!(closed, "closed established connections for daemon shutdown");
2863        }
2864        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
2865        // already completed and must not be polled again; the child shutdown
2866        // wait is told it is escalated and gets a future that never fires.
2867        let escalated_here = escalated && !already_escalated;
2868        let remaining_escalate = async move {
2869            if escalated_here {
2870                std::future::pending::<()>().await;
2871            } else {
2872                escalate.await;
2873            }
2874        };
2875        crate::child_roster::end_children_for_daemon_shutdown(
2876            &self.child_roster,
2877            escalated,
2878            remaining_escalate,
2879        )
2880        .await;
2881    }
2882
2883    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2884        Self {
2885            registry,
2886            restart_policy,
2887            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2888            connection_file_path: None,
2889            capture_logs_dir: None,
2890            forwarding: None,
2891            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2892            supervisor_handle: None,
2893            health: HealthConfig::default(),
2894            daemon_start_clock: crate::clock::StartClock::capture(),
2895            terminal_journal: None,
2896            spawn_events: SpawnEventFeed::default(),
2897            provenance_probe: ExecutableIdentityProbe::default(),
2898            child_roster: ChildRoster::default(),
2899            #[cfg(target_os = "linux")]
2900            cgroup_placement: None,
2901            #[cfg(test)]
2902            test_after_first_spawn: AfterFirstSpawnHook::default(),
2903        }
2904    }
2905
2906    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
2907        self.drain_timeout = drain_timeout;
2908        self
2909    }
2910
2911    pub fn with_process_liveness(
2912        mut self,
2913        process_liveness: Arc<SupervisorProcessLiveness>,
2914    ) -> Self {
2915        self.process_liveness = process_liveness;
2916        self
2917    }
2918
2919    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
2920        self.connection_file_path = Some(connection_file_path.into());
2921        self
2922    }
2923
2924    /// Enables daemon-owned capture files for supervised stdout and stderr.
2925    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
2926        self.capture_logs_dir = Some(logs_dir.into());
2927        self
2928    }
2929
2930    /// Names this daemon lifetime in spawn events, independently of whether a
2931    /// terminal journal is configured.
2932    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
2933        // A millisecond start stamp can repeat after clock rollback or a rapid
2934        // restart. Use the connection file's random daemon_id instead: it already
2935        // identifies this daemon lifetime independently of the wall clock.
2936        self.spawn_events.configure_incarnation(daemon_incarnation);
2937        self
2938    }
2939
2940    /// Enables best-effort history shared by every supervised module. Without
2941    /// it, terminal history is kept only in each module's in-memory ring.
2942    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
2943        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
2944        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
2945            path,
2946            daemon_incarnation,
2947        )));
2948        this
2949    }
2950
2951    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
2952        self.forwarding = Some(forwarding);
2953        self
2954    }
2955
2956    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
2957        self.spawn_events = supervisor_handle.spawn_events.clone();
2958        self.supervisor_handle = Some(supervisor_handle);
2959        self
2960    }
2961
2962    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
2963        self.health = health;
2964        self
2965    }
2966
2967    /// Keeps a record of every live child at `path`, rewritten on each spawn and
2968    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
2969    /// record is kept.
2970    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
2971        self.child_roster.record_to(path.into());
2972        self
2973    }
2974
2975    #[cfg(target_os = "linux")]
2976    pub fn with_cgroup_placement(
2977        mut self,
2978        cgroup_placement: Option<subc_cgroup::Placement>,
2979    ) -> Self {
2980        self.cgroup_placement = cgroup_placement;
2981        self
2982    }
2983
2984    /// Spawn `spec.program` and start monitoring it.
2985    ///
2986    /// The child is expected to parse `--subc <connection-file-path>`, read the
2987    /// TCP+key connection file, authenticate to the already-running listener, and
2988    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
2989    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
2990        validate_spec(&spec)?;
2991        self.establish_identity(&spec);
2992
2993        let runtime = self.runtime_config();
2994        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
2995        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
2996        let spawned = spawn_child(
2997            &spec,
2998            runtime.connection_file_path.as_deref(),
2999            self.supervisor_handle.as_ref(),
3000            &runtime.stderr_ring,
3001            runtime.capture_logs_dir.as_deref(),
3002            &runtime.child_roster,
3003            #[cfg(target_os = "linux")]
3004            runtime.cgroup_placement.as_ref(),
3005        );
3006        #[cfg(test)]
3007        self.test_after_first_spawn.run(&spec.module_id);
3008        let child = match spawned {
3009            Ok(child) => child,
3010            Err(err) => {
3011                // Unlike the configured paths, a failed `spawn` leaves nothing
3012                // on the roster, so the module must not stay marked configured.
3013                self.abandon_unrostered(&spec.module_id);
3014                return Err(err);
3015            }
3016        };
3017        self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3018        if let Some(spawn) = activity_spawn {
3019            spawn.complete(&snapshot);
3020        }
3021
3022        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3023    }
3024
3025    /// Make `spec`'s module count as configured, with its identity gates
3026    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3027    /// exists.
3028    ///
3029    /// Every path that takes on a new module calls this before `spawn_child`.
3030    /// The order is the point: the child can connect, register, sync its
3031    /// scopes and ask about them as soon as it is spawned, and the module is
3032    /// only put on the roster after `spawn_child` returns. Were the mark set
3033    /// with the roster entry, a fast child would see its own owner reported
3034    /// as not configured, and a scoped `route.open` in that window would be
3035    /// refused as terminal `scope_not_live` ("will never sync") instead of
3036    /// retryable `scope_not_synced`.
3037    fn establish_identity(&self, spec: &ModuleSpec) {
3038        if let Some(supervisor_handle) = &self.supervisor_handle {
3039            supervisor_handle.apply_identity_configuration(spec);
3040            supervisor_handle.mark_configured(&spec.module_id);
3041        }
3042    }
3043
3044    /// Take back [`Self::establish_identity`]'s configured mark when the
3045    /// module will not be put on the roster after all.
3046    fn abandon_unrostered(&self, module_id: &str) {
3047        if let Some(supervisor_handle) = &self.supervisor_handle {
3048            supervisor_handle.unmark_configured_unless_rostered(module_id);
3049        }
3050    }
3051
3052    /// Record a freshly spawned first process as running. On failure the
3053    /// module never reaches the roster, so its configured mark is taken back.
3054    fn mark_first_process_running(
3055        &self,
3056        spec: &ModuleSpec,
3057        runtime: &SupervisorRuntimeConfig,
3058        snapshot: &SharedSnapshot,
3059        child: &SupervisedChild,
3060    ) -> Result<(), SuperviseError> {
3061        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3062            self.abandon_unrostered(&spec.module_id);
3063            return Err(err);
3064        }
3065        self.process_liveness
3066            .track(spec.module_id.clone(), Arc::clone(snapshot));
3067        Ok(())
3068    }
3069
3070    /// Start supervising a module declared in daemon configuration.
3071    ///
3072    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3073    /// failures in the supervisor handle so operator-facing `supervisor.list`
3074    /// reflects every configured module while daemon startup continues.
3075    pub fn supervise_configured(
3076        &self,
3077        spec: ModuleSpec,
3078        enabled: bool,
3079    ) -> Result<SupervisedModule, SuperviseError> {
3080        self.supervise_configured_requested(
3081            spec,
3082            enabled,
3083            self.health.clone(),
3084            None,
3085            self.restart_policy,
3086            None,
3087        )
3088    }
3089
3090    /// Supervise a configured module with its own health, drain, and crash
3091    /// budget. The restart policy is per-module because the config file is:
3092    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3093    /// module that is expensive to restart should not be forced onto the same
3094    /// budget as one that is cheap.
3095    pub fn supervise_configured_with_health(
3096        &self,
3097        spec: ModuleSpec,
3098        enabled: bool,
3099        health: HealthConfig,
3100        drain_timeout_ms: Option<u64>,
3101        restart_policy: RestartPolicy,
3102    ) -> Result<SupervisedModule, SuperviseError> {
3103        self.supervise_configured_requested(
3104            spec,
3105            enabled,
3106            health,
3107            drain_timeout_ms,
3108            restart_policy,
3109            None,
3110        )
3111    }
3112
3113    pub(crate) fn supervise_configured_requested(
3114        &self,
3115        spec: ModuleSpec,
3116        enabled: bool,
3117        health: HealthConfig,
3118        drain_timeout_ms: Option<u64>,
3119        restart_policy: RestartPolicy,
3120        requester: Option<Principal>,
3121    ) -> Result<SupervisedModule, SuperviseError> {
3122        let registered = self.register_configured_with_health(
3123            spec,
3124            enabled,
3125            health,
3126            drain_timeout_ms,
3127            restart_policy,
3128        )?;
3129        self.start_registered_configured(registered, requester)
3130    }
3131
3132    /// Record a configured module as starting or disabled, without launching a
3133    /// process or running its monitor. Bootstrap registers the entire roster
3134    /// before publishing discovery, then activates it once children can read
3135    /// the connection file passed in their `--subc` argument.
3136    pub(crate) fn register_configured_with_health(
3137        &self,
3138        spec: ModuleSpec,
3139        enabled: bool,
3140        health: HealthConfig,
3141        drain_timeout_ms: Option<u64>,
3142        restart_policy: RestartPolicy,
3143    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3144        validate_spec(&spec)?;
3145        self.establish_identity(&spec);
3146
3147        let mut runtime = self.runtime_config();
3148        runtime.health = health;
3149        runtime.restart_policy = restart_policy;
3150        if let Some(ms) = drain_timeout_ms {
3151            runtime.drain_timeout = Duration::from_millis(ms);
3152            *runtime
3153                .effective_drain_timeout
3154                .lock()
3155                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3156        }
3157        let snapshot = if enabled {
3158            SupervisorSnapshot::starting()
3159        } else {
3160            SupervisorSnapshot::disabled()
3161        };
3162        let snapshot = {
3163            let mut snapshot = snapshot;
3164            if enabled {
3165                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3166                {
3167                    snapshot.waiting_for_previous_pid = Some(pid);
3168                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3169                }
3170            }
3171            snapshot
3172        };
3173        let (module, commands) =
3174            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3175        Ok(RegisteredConfiguredModule {
3176            spec,
3177            runtime,
3178            module,
3179            commands,
3180        })
3181    }
3182
3183    /// Activate a previously registered module. Queued control requests are
3184    /// served by the monitor after the initial configured spawn has settled.
3185    pub(crate) fn start_registered_configured(
3186        &self,
3187        registered: RegisteredConfiguredModule,
3188        requester: Option<Principal>,
3189    ) -> Result<SupervisedModule, SuperviseError> {
3190        let RegisteredConfiguredModule {
3191            spec,
3192            mut runtime,
3193            module,
3194            commands,
3195        } = registered;
3196        runtime.start_requester = requester;
3197        let snapshot = &module.inner.snapshot;
3198        let should_spawn = {
3199            let state = lock_snapshot(snapshot)?;
3200            state.enabled && state.waiting_for_previous_pid.is_none()
3201        };
3202        let child = if should_spawn {
3203            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3204            let spawned = spawn_child(
3205                &spec,
3206                runtime.connection_file_path.as_deref(),
3207                self.supervisor_handle.as_ref(),
3208                &runtime.stderr_ring,
3209                runtime.capture_logs_dir.as_deref(),
3210                &runtime.child_roster,
3211                #[cfg(target_os = "linux")]
3212                runtime.cgroup_placement.as_ref(),
3213            );
3214            #[cfg(test)]
3215            self.test_after_first_spawn.run(&spec.module_id);
3216            match spawned {
3217                Ok(child) => {
3218                    self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3219                    if let Some(spawn) = activity_spawn {
3220                        spawn.complete(snapshot);
3221                    }
3222                    Some(child)
3223                }
3224                Err(err) => {
3225                    if runtime.health.critical {
3226                        error!(
3227                            module_id = %spec.module_id,
3228                            program = %spec.program.display(),
3229                            error = %err,
3230                            "critical configured module failed to spawn; marking failed and alerting"
3231                        );
3232                    } else {
3233                        error!(
3234                            module_id = %spec.module_id,
3235                            program = %spec.program.display(),
3236                            error = %err,
3237                            "configured module failed to spawn; marking failed and continuing"
3238                        );
3239                    }
3240                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
3241                        state.state = ModuleState::Failed;
3242                        clear_current_process_facts(state);
3243                    })?;
3244                    None
3245                }
3246            }
3247        } else {
3248            None
3249        };
3250        self.start_monitor(spec, runtime, &module, child, commands);
3251        Ok(module)
3252    }
3253
3254    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3255        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3256        SupervisorRuntimeConfig {
3257            start_requester: None,
3258            pending_activity_restart: false,
3259            provenance_probe: self.provenance_probe.clone(),
3260            scheduled_respawn: Arc::default(),
3261            deferred_reload_reply: Arc::default(),
3262            restart_policy: self.restart_policy,
3263            drain_timeout: self.drain_timeout,
3264            // Shared with this module's roster copy: daemon shutdown waits on
3265            // each child for the module's own drain budget, as resolved now.
3266            child_roster: self
3267                .child_roster
3268                .for_module(Arc::clone(&effective_drain_timeout)),
3269            effective_drain_timeout,
3270            default_drain_timeout: self.drain_timeout,
3271            health: self.health.clone(),
3272            connection_file_path: self.connection_file_path.clone(),
3273            capture_logs_dir: self.capture_logs_dir.clone(),
3274            forwarding: self.forwarding.clone(),
3275            supervisor_handle: self.supervisor_handle.clone(),
3276            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3277            terminal_ring: Arc::new(Mutex::new(
3278                TerminalRing::new(
3279                    TerminalRingConfig::default(),
3280                    self.daemon_start_clock.started_at_ms(),
3281                )
3282                .with_start_clock(self.daemon_start_clock)
3283                .with_journal(self.terminal_journal.clone())
3284                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3285            )),
3286            spawn_events: self.spawn_events.clone(),
3287            #[cfg(target_os = "linux")]
3288            cgroup_placement: self.cgroup_placement.clone(),
3289            #[cfg(test)]
3290            test_seed_stale_facts_before_enable_spawn: false,
3291            #[cfg(test)]
3292            test_reload_exit_record_gate: None,
3293        }
3294    }
3295
3296    fn supervised_module(
3297        &self,
3298        spec: ModuleSpec,
3299        runtime: SupervisorRuntimeConfig,
3300        snapshot: SharedSnapshot,
3301        child: Option<SupervisedChild>,
3302    ) -> SupervisedModule {
3303        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3304        self.start_monitor(spec, runtime, &module, child, commands);
3305        module
3306    }
3307
3308    fn register_module(
3309        &self,
3310        spec: &ModuleSpec,
3311        runtime: &SupervisorRuntimeConfig,
3312        snapshot: SharedSnapshot,
3313    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3314        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3315            spec: spec.clone(),
3316            health: runtime.health.clone(),
3317        }));
3318        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3319        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3320        // The module's OWN policy, which may be its per-module config rather than
3321        // the supervisor-wide one; status must report the budget the supervise
3322        // loop actually enforces.
3323        let restart_policy = runtime.restart_policy;
3324        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3325        let (tx, rx) = mpsc::channel(4);
3326        let module_id = spec.module_id.clone();
3327        let module = SupervisedModule {
3328            inner: Arc::new(SupervisedModuleInner {
3329                module_id: module_id.clone(),
3330                registry: Arc::clone(&self.registry),
3331                snapshot,
3332                configuration,
3333                stderr_ring,
3334                terminal_ring,
3335                commands: tx,
3336                monitor: Mutex::new(None),
3337                restart_policy,
3338                effective_drain_timeout,
3339                provenance_probe: self.provenance_probe.clone(),
3340            }),
3341        };
3342        // Identity gates and the configured mark must precede the roster entry,
3343        // so a client never sees a module without its admission policy.
3344        if let Some(supervisor_handle) = &self.supervisor_handle {
3345            supervisor_handle.insert(module.clone());
3346        }
3347        (module, rx)
3348    }
3349
3350    fn start_monitor(
3351        &self,
3352        spec: ModuleSpec,
3353        runtime: SupervisorRuntimeConfig,
3354        module: &SupervisedModule,
3355        child: Option<SupervisedChild>,
3356        commands: mpsc::Receiver<SupervisorCommand>,
3357    ) {
3358        let monitor = tokio::spawn(supervise_loop(
3359            spec,
3360            runtime,
3361            Arc::clone(&self.registry),
3362            Arc::clone(&self.process_liveness),
3363            Arc::clone(&module.inner.snapshot),
3364            child,
3365            commands,
3366        ));
3367        *module
3368            .inner
3369            .monitor
3370            .lock()
3371            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3372    }
3373}
3374
3375impl Default for Supervisor {
3376    fn default() -> Self {
3377        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3378    }
3379}
3380
3381/// Handle to one supervised child process.
3382#[derive(Clone)]
3383pub struct SupervisedModule {
3384    inner: Arc<SupervisedModuleInner>,
3385}
3386
3387struct SupervisedModuleInner {
3388    module_id: String,
3389    registry: Arc<Registry>,
3390    snapshot: SharedSnapshot,
3391    configuration: Arc<Mutex<SupervisedConfiguration>>,
3392    stderr_ring: Arc<Mutex<StderrRing>>,
3393    terminal_ring: Arc<Mutex<TerminalRing>>,
3394    commands: mpsc::Sender<SupervisorCommand>,
3395    monitor: Mutex<Option<JoinHandle<()>>>,
3396    /// Copied from the supervisor's runtime config at spawn so `status()` can
3397    /// report the restart budget without reaching back into the supervisor. The
3398    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3399    restart_policy: RestartPolicy,
3400    effective_drain_timeout: Arc<Mutex<Duration>>,
3401    provenance_probe: ExecutableIdentityProbe,
3402}
3403
3404impl fmt::Debug for SupervisedModule {
3405    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3406        f.debug_struct("SupervisedModule")
3407            .field("module_id", &self.inner.module_id)
3408            .field("status", &self.status())
3409            .finish_non_exhaustive()
3410    }
3411}
3412
3413impl SupervisedModule {
3414    pub fn module_id(&self) -> &str {
3415        &self.inner.module_id
3416    }
3417
3418    /// Test-only: put one probe miss on the streak, the way
3419    /// `handle_health_probe_failure` does, so tests can assert what a later
3420    /// event does to the streak without driving the whole probe loop.
3421    #[cfg(test)]
3422    pub(crate) fn record_health_probe_failure_for_test(
3423        &self,
3424        detail: &str,
3425    ) -> Result<(), SuperviseError> {
3426        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3427            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3428            state.health.detail = Some(detail.to_string());
3429        })
3430    }
3431
3432    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3433        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3434    }
3435
3436    pub(crate) fn registration_end_reason(
3437        &self,
3438    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3439        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3440        Ok(match snapshot.state {
3441            ModuleState::Draining if snapshot.draining_to_replace => {
3442                Some(RegistrationEndReason::SupervisorRestart)
3443            }
3444            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3445            ModuleState::Restarting if snapshot.draining_to_replace => {
3446                Some(RegistrationEndReason::SupervisorRestart)
3447            }
3448            _ => None,
3449        })
3450    }
3451
3452    /// The module's retained stderr, newest lines last.
3453    ///
3454    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3455    /// module, `supervisor.list` renders every module, and putting it in the
3456    /// shared snapshot would make each status read carry a payload almost nobody
3457    /// asked for. Callers that want the text ask for it.
3458    pub fn stderr_tail(
3459        &self,
3460        max_lines: Option<usize>,
3461        max_bytes: Option<usize>,
3462    ) -> StderrTailSnapshot {
3463        self.inner
3464            .stderr_ring
3465            .lock()
3466            .unwrap_or_else(|poisoned| poisoned.into_inner())
3467            .snapshot(max_lines, max_bytes)
3468    }
3469
3470    /// The module's bounded terminal history, oldest retained exit first.
3471    ///
3472    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3473    /// daemon whose in-memory history was necessarily reset.
3474    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3475        self.inner
3476            .terminal_ring
3477            .lock()
3478            .unwrap_or_else(|poisoned| poisoned.into_inner())
3479            .snapshot()
3480    }
3481
3482    /// Retained observations from the current ring and all journal generations.
3483    ///
3484    /// Blocking: this reads the journal files. Async callers use
3485    /// [`Self::read_durable_terminal_history`].
3486    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3487        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3488    }
3489
3490    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3491    /// read (up to every retained generation) never occupies a runtime worker.
3492    /// Fails only if the blocking task could not finish (runtime shutdown or a
3493    /// panic in the read).
3494    pub(crate) async fn read_durable_terminal_history(
3495        &self,
3496    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3497        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3498        let module_id = self.inner.module_id.clone();
3499        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3500            .await
3501    }
3502
3503    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3504        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3505            .map(|(status, _)| status)
3506    }
3507
3508    pub(crate) fn record_deliberate_severance(
3509        &self,
3510        identity: ProcessIdentity,
3511    ) -> Result<bool, SuperviseError> {
3512        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3513        if snapshot.pid != Some(identity.pid)
3514            || snapshot.process_start_time != Some(identity.start_time)
3515        {
3516            return Ok(false);
3517        }
3518        snapshot.deliberate_severance = Some(identity);
3519        Ok(true)
3520    }
3521
3522    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3523    ///
3524    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3525    /// does not produce reader-observability logs.
3526    pub(crate) fn status_for_control(
3527        &self,
3528        caller: &'static str,
3529    ) -> Result<ModuleStatus, SuperviseError> {
3530        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3531            .map(|(status, _)| status)
3532    }
3533
3534    fn status_with_snapshot_lock(
3535        &self,
3536        snapshot: &SharedSnapshot,
3537        caller: Option<&'static str>,
3538    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3539        let mut guard = match caller {
3540            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3541            None => lock_snapshot(snapshot)?,
3542        };
3543        // Read the budget through the pruning path so a reader sees the same
3544        // in-window count the restart decision would use, not a stale total.
3545        let restart_count =
3546            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3547        let snapshot = guard.clone();
3548        drop(guard);
3549        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3550            SuperviseError::StatePoisoned {
3551                module_id: Some(self.inner.module_id.clone()),
3552            }
3553        })?;
3554        let registration_active = self
3555            .inner
3556            .registry
3557            .get_module(&self.inner.module_id)
3558            .map_err(SuperviseError::Registry)?
3559            .is_some();
3560        let protocol = snapshot
3561            .spawned_protocol
3562            .unwrap_or(self.declared_protocol()?);
3563        let running_process =
3564            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3565        // Registration is the difference between the two protocols and the only
3566        // one: a subc module that has not registered cannot serve a request even
3567        // though its process is up, and a `none` module never registers at all,
3568        // so requiring it there would pin `live` to false for the whole life of
3569        // a perfectly healthy process.
3570        let live = match protocol {
3571            ModuleProtocol::Subc => running_process && registration_active,
3572            ModuleProtocol::None => running_process,
3573        };
3574
3575        Ok((
3576            ModuleStatus {
3577                waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3578                waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3579                module_id: self.inner.module_id.clone(),
3580                launch_nonce_source: observed_launch_nonce_source(&snapshot),
3581                state: snapshot.state,
3582                enabled: snapshot.enabled,
3583                process_alive: snapshot.process_alive,
3584                registration_active,
3585                protocol,
3586                live,
3587                restart_count,
3588                lifetime_restarts: snapshot.lifetime_restarts,
3589                spawn_generation: snapshot.spawn_generation,
3590                max_restarts: self.inner.restart_policy.max_restarts,
3591                restart_window: self.inner.restart_policy.window,
3592                drain_timeout,
3593                restart_backoff: self.inner.restart_policy.backoff,
3594                restart_max_backoff: self.inner.restart_policy.max_backoff,
3595                pid: snapshot.reported_pid(),
3596                spawned_at_ms: snapshot.spawned_at_ms,
3597                spawned_from: snapshot.spawned_from,
3598                process_start_time: snapshot.process_start_time,
3599                last_exit: snapshot.last_exit,
3600                health: snapshot.health,
3601            },
3602            snapshot.spawned_file_identity.clone(),
3603        ))
3604    }
3605
3606    #[cfg(test)]
3607    pub(crate) fn hold_snapshot_for_test(
3608        &self,
3609        acquired: std::sync::mpsc::Sender<()>,
3610        hold: Duration,
3611    ) -> std::thread::JoinHandle<()> {
3612        let snapshot = Arc::clone(&self.inner.snapshot);
3613        std::thread::spawn(move || {
3614            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3615            acquired
3616                .send(())
3617                .expect("test receiver waits for snapshot lock");
3618            std::thread::sleep(hold);
3619        })
3620    }
3621
3622    /// The status and the running-image check for `supervisor.provenance`,
3623    /// taken from one status read. The exec acknowledgement can land between
3624    /// two separate reads, and the reply would then pair "no pid yet" with an
3625    /// image observed after the module started, which describes no single
3626    /// moment.
3627    pub(crate) async fn status_and_running_image_agreement(
3628        &self,
3629    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3630        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3631        let image = self
3632            .inner
3633            .provenance_probe
3634            .observe(
3635                status.pid,
3636                status.spawned_from.as_deref(),
3637                identity,
3638                status.process_start_time,
3639            )
3640            .await;
3641        Ok((status, image))
3642    }
3643
3644    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3645        let snapshot = match lock_snapshot(&self.inner.snapshot) {
3646            Ok(snapshot) => snapshot.clone(),
3647            Err(_) => {
3648                return subc_control::RunningImageAgreement::Unavailable {
3649                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
3650                };
3651            }
3652        };
3653        self.inner
3654            .provenance_probe
3655            .observe(
3656                snapshot.reported_pid(),
3657                snapshot.spawned_from.as_deref(),
3658                snapshot.spawned_file_identity.clone(),
3659                snapshot.process_start_time,
3660            )
3661            .await
3662    }
3663
3664    /// Memory and CPU time of the module's current process, read now. Only the
3665    /// process the supervisor spawned is read, not processes it has started.
3666    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3667        #[cfg(windows)]
3668        {
3669            let image = lock_snapshot(&self.inner.snapshot)
3670                .ok()
3671                .and_then(|snapshot| {
3672                    snapshot
3673                        .spawned_file_identity
3674                        .as_ref()
3675                        .and_then(|identity| identity.windows_image.clone())
3676                });
3677            match image {
3678                Some(image) => {
3679                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
3680                }
3681                None => subc_control::ChildResourceUsage::Unavailable {
3682                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3683                },
3684            }
3685        }
3686        #[cfg(not(windows))]
3687        {
3688            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3689                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3690                Err(_) => {
3691                    return subc_control::ChildResourceUsage::Unavailable {
3692                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3693                    }
3694                }
3695            };
3696            crate::child_resources::read(pid, start_time)
3697        }
3698    }
3699
3700    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3701        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3702        Ok(match snapshot.state {
3703            ModuleState::Restarting => true,
3704            ModuleState::Failed | ModuleState::Disabled => false,
3705            _ if snapshot.enabled && snapshot.respawn_pending => true,
3706            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3707        })
3708    }
3709
3710    #[cfg(test)]
3711    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3712        self.is_warming_with_snapshot_lock(None)
3713    }
3714
3715    pub(crate) fn is_warming_for_control(
3716        &self,
3717        caller: &'static str,
3718    ) -> Result<bool, SuperviseError> {
3719        self.is_warming_with_snapshot_lock(Some(caller))
3720    }
3721
3722    fn is_warming_with_snapshot_lock(
3723        &self,
3724        caller: Option<&'static str>,
3725    ) -> Result<bool, SuperviseError> {
3726        let snapshot = match caller {
3727            Some(caller) => {
3728                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3729            }
3730            None => lock_snapshot(&self.inner.snapshot)?,
3731        }
3732        .clone();
3733        Ok(matches!(
3734            snapshot.state,
3735            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3736        ))
3737    }
3738
3739    /// Drain the module and stop monitoring it.
3740    pub async fn drain(&self) -> Result<(), SuperviseError> {
3741        self.stop().await
3742    }
3743
3744    pub(crate) async fn retire_requested(
3745        &self,
3746        requester: Option<Principal>,
3747    ) -> Result<(), SuperviseError> {
3748        match self.state()? {
3749            ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3750            ModuleState::Starting
3751            | ModuleState::Running
3752            | ModuleState::Unresponsive
3753            | ModuleState::Restarting
3754            | ModuleState::Draining
3755            | ModuleState::Disabled => {}
3756        }
3757
3758        let (reply_tx, reply_rx) = oneshot::channel();
3759        self.inner
3760            .commands
3761            .send(SupervisorCommand::Retire {
3762                requester,
3763                reply: reply_tx,
3764            })
3765            .await
3766            .map_err(|_| SuperviseError::CommandClosed {
3767                module_id: self.inner.module_id.clone(),
3768            })?;
3769        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3770            module_id: self.inner.module_id.clone(),
3771        })?
3772    }
3773
3774    pub async fn stop(&self) -> Result<(), SuperviseError> {
3775        self.stop_requested(None).await
3776    }
3777
3778    pub(crate) async fn stop_requested(
3779        &self,
3780        requester: Option<Principal>,
3781    ) -> Result<(), SuperviseError> {
3782        match self.state()? {
3783            ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3784            ModuleState::Starting
3785            | ModuleState::Running
3786            | ModuleState::Unresponsive
3787            | ModuleState::Restarting
3788            | ModuleState::Draining
3789            | ModuleState::Disabled => {}
3790        }
3791
3792        let (reply_tx, reply_rx) = oneshot::channel();
3793        self.inner
3794            .commands
3795            .send(SupervisorCommand::Drain {
3796                requester,
3797                reply: reply_tx,
3798            })
3799            .await
3800            .map_err(|_| SuperviseError::CommandClosed {
3801                module_id: self.inner.module_id.clone(),
3802            })?;
3803        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3804            module_id: self.inner.module_id.clone(),
3805        })?
3806    }
3807
3808    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3809        self.restart_requested(drain_timeout_ms, None).await
3810    }
3811
3812    pub(crate) async fn restart_requested(
3813        &self,
3814        drain_timeout_ms: Option<u64>,
3815        requester: Option<Principal>,
3816    ) -> Result<(), SuperviseError> {
3817        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3818        let (reply_tx, reply_rx) = oneshot::channel();
3819        self.inner
3820            .commands
3821            .send(SupervisorCommand::Restart {
3822                requester,
3823                drain_timeout_ms,
3824                received_at_generation,
3825                queued_at: Instant::now(),
3826                reply: reply_tx,
3827            })
3828            .await
3829            .map_err(|_| SuperviseError::CommandClosed {
3830                module_id: self.inner.module_id.clone(),
3831            })?;
3832        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3833            module_id: self.inner.module_id.clone(),
3834        })?
3835    }
3836
3837    /// Blue/green restart: see [`SupervisorCommand::Swap`] and the
3838    /// `supervisor_swap` module. Returns once the swap has cut over (the old
3839    /// process then drains in the background of the supervise loop) or has
3840    /// failed, leaving the old process serving.
3841    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3842        self.swap_requested(ready_timeout, None).await
3843    }
3844
3845    pub(crate) async fn swap_requested(
3846        &self,
3847        ready_timeout: Option<Duration>,
3848        requester: Option<Principal>,
3849    ) -> Result<(), SuperviseError> {
3850        let (reply_tx, reply_rx) = oneshot::channel();
3851        self.inner
3852            .commands
3853            .send(SupervisorCommand::Swap {
3854                requester,
3855                ready_timeout,
3856                reply: reply_tx,
3857            })
3858            .await
3859            .map_err(|_| SuperviseError::CommandClosed {
3860                module_id: self.inner.module_id.clone(),
3861            })?;
3862        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3863            module_id: self.inner.module_id.clone(),
3864        })?
3865    }
3866
3867    pub async fn reload(&self) -> Result<(), SuperviseError> {
3868        self.reload_requested(None).await
3869    }
3870
3871    pub(crate) async fn reload_requested(
3872        &self,
3873        requester: Option<Principal>,
3874    ) -> Result<(), SuperviseError> {
3875        let (reply_tx, reply_rx) = oneshot::channel();
3876        self.inner
3877            .commands
3878            .send(SupervisorCommand::Reload {
3879                requester,
3880                reply: reply_tx,
3881            })
3882            .await
3883            .map_err(|_| SuperviseError::CommandClosed {
3884                module_id: self.inner.module_id.clone(),
3885            })?;
3886        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3887            module_id: self.inner.module_id.clone(),
3888        })?
3889    }
3890
3891    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3892        self.set_enabled_requested(enabled, None).await
3893    }
3894
3895    pub(crate) async fn set_enabled_requested(
3896        &self,
3897        enabled: bool,
3898        requester: Option<Principal>,
3899    ) -> Result<bool, SuperviseError> {
3900        let (reply_tx, reply_rx) = oneshot::channel();
3901        self.inner
3902            .commands
3903            .send(SupervisorCommand::SetEnabled {
3904                requester,
3905                enabled,
3906                reply: reply_tx,
3907            })
3908            .await
3909            .map_err(|_| SuperviseError::CommandClosed {
3910                module_id: self.inner.module_id.clone(),
3911            })?;
3912        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3913            module_id: self.inner.module_id.clone(),
3914        })?
3915    }
3916
3917    /// The current process's protocol, or the configured protocol when down.
3918    /// A rescan stores the next launch spec without changing how an existing
3919    /// process registers, serves routes, is probed, or exits.
3920    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
3921        let configured = self
3922            .inner
3923            .configuration
3924            .lock()
3925            .map_err(|_| SuperviseError::StatePoisoned {
3926                module_id: Some(self.inner.module_id.clone()),
3927            })?
3928            .spec
3929            .protocol;
3930        let state = lock_snapshot(&self.inner.snapshot)?;
3931        Ok(state.spawned_protocol.unwrap_or(configured))
3932    }
3933
3934    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
3935        let configuration =
3936            self.inner
3937                .configuration
3938                .lock()
3939                .map_err(|_| SuperviseError::StatePoisoned {
3940                    module_id: Some(self.inner.module_id.clone()),
3941                })?;
3942        Ok((configuration.spec.clone(), configuration.health.clone()))
3943    }
3944
3945    /// Replace this module's launch spec, keeping its health and drain policy,
3946    /// the way a rescan does for a changed config entry. The running process is
3947    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
3948    #[cfg(any(test, feature = "test-support"))]
3949    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
3950        let (_, health) = self.configuration()?;
3951        let drain_timeout_ms = u64::try_from(
3952            self.inner
3953                .effective_drain_timeout
3954                .lock()
3955                .unwrap_or_else(|poisoned| poisoned.into_inner())
3956                .as_millis(),
3957        )
3958        .ok();
3959        self.update_configuration(spec, health, drain_timeout_ms)
3960            .await
3961    }
3962
3963    pub(crate) async fn update_configuration(
3964        &self,
3965        spec: ModuleSpec,
3966        health: HealthConfig,
3967        drain_timeout_ms: Option<u64>,
3968    ) -> Result<(), SuperviseError> {
3969        if spec.module_id != self.inner.module_id {
3970            return Err(SuperviseError::InvalidSpec {
3971                reason: "a supervised module's module_id cannot be changed".to_string(),
3972            });
3973        }
3974        validate_spec(&spec)?;
3975        let (reply_tx, reply_rx) = oneshot::channel();
3976        self.inner
3977            .commands
3978            .send(SupervisorCommand::UpdateConfiguration {
3979                spec: spec.clone(),
3980                health: health.clone(),
3981                drain_timeout_ms,
3982                reply: reply_tx,
3983            })
3984            .await
3985            .map_err(|_| SuperviseError::CommandClosed {
3986                module_id: self.inner.module_id.clone(),
3987            })?;
3988        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3989            module_id: self.inner.module_id.clone(),
3990        })?;
3991        let mut configuration =
3992            self.inner
3993                .configuration
3994                .lock()
3995                .map_err(|_| SuperviseError::StatePoisoned {
3996                    module_id: Some(self.inner.module_id.clone()),
3997                })?;
3998        configuration.spec = spec;
3999        configuration.health = health;
4000        Ok(())
4001    }
4002}
4003
4004impl Drop for SupervisedModuleInner {
4005    fn drop(&mut self) {
4006        let Ok(mut monitor) = self.monitor.lock() else {
4007            return;
4008        };
4009        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4010            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4011                state.state = ModuleState::Stopped;
4012                clear_current_process_facts(state);
4013            });
4014            monitor.abort();
4015        }
4016        let _ = monitor.take();
4017    }
4018}
4019
4020#[derive(Debug)]
4021enum SupervisorCommand {
4022    Drain {
4023        requester: Option<Principal>,
4024        reply: oneshot::Sender<Result<(), SuperviseError>>,
4025    },
4026    Retire {
4027        requester: Option<Principal>,
4028        reply: oneshot::Sender<Result<(), SuperviseError>>,
4029    },
4030    Restart {
4031        requester: Option<Principal>,
4032        /// Operator override for this one restart's drain budget, in ms. `None`
4033        /// uses the module's configured/default budget; `Some(0)` cuts
4034        /// immediately (wedge bounce: a stuck request never settles, so
4035        /// waiting only delays recovery).
4036        drain_timeout_ms: Option<u64>,
4037        /// The module's `spawn_generation` when the request was received, before
4038        /// it waited in the command queue. A queued restart whose module has
4039        /// since spawned a newer process is already satisfied (see the handler).
4040        received_at_generation: u64,
4041        /// When the request entered the command queue, so the handler can log
4042        /// how long it waited behind the loop's other work.
4043        queued_at: Instant,
4044        reply: oneshot::Sender<Result<(), SuperviseError>>,
4045    },
4046    Reload {
4047        requester: Option<Principal>,
4048        reply: oneshot::Sender<Result<(), SuperviseError>>,
4049    },
4050    SetEnabled {
4051        requester: Option<Principal>,
4052        enabled: bool,
4053        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4054    },
4055    UpdateConfiguration {
4056        spec: ModuleSpec,
4057        health: HealthConfig,
4058        /// Per-module drain override from the new config; `None` re-resolves to
4059        /// the supervisor-wide default.
4060        drain_timeout_ms: Option<u64>,
4061        reply: oneshot::Sender<()>,
4062    },
4063    Swap {
4064        requester: Option<Principal>,
4065        /// How long the candidate may take to register and declare itself
4066        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4067        ready_timeout: Option<Duration>,
4068        /// Answered at cutover or failure; the incumbent's drain follows.
4069        reply: oneshot::Sender<Result<(), SuperviseError>>,
4070    },
4071}
4072
4073#[derive(Debug)]
4074pub enum SuperviseError {
4075    InvalidSpec {
4076        reason: String,
4077    },
4078    Spawn {
4079        program: PathBuf,
4080        source: io::Error,
4081        cgroup_path: Option<PathBuf>,
4082    },
4083    Cgroup {
4084        module_id: String,
4085        source: io::Error,
4086    },
4087    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4088    /// than spawn a reserved module without its identity binding.
4089    LaunchNonce {
4090        reason: String,
4091    },
4092    Wait {
4093        module_id: String,
4094        source: io::Error,
4095    },
4096    Kill {
4097        module_id: String,
4098        source: io::Error,
4099    },
4100    Forwarding(ForwardingError),
4101    Registry(RegistryError),
4102    ReloadUnavailable {
4103        module_id: String,
4104        reason: String,
4105    },
4106    /// An operator restart/reload was requested for a module that is currently
4107    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4108    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4109    /// by a restart, so these commands are rejected instead of re-enabling it.
4110    Disabled {
4111        module_id: String,
4112    },
4113    ReloadFailed {
4114        module_id: String,
4115        reason: String,
4116    },
4117    RegistrationStillActive {
4118        module_id: String,
4119        waited: Duration,
4120    },
4121    StatePoisoned {
4122        module_id: Option<String>,
4123    },
4124    CommandClosed {
4125        module_id: String,
4126    },
4127    /// A restart or reload arrived while a swap's candidate was warming. The
4128    /// swap owns the module until it cuts over or fails; a stop or disable
4129    /// would have aborted it instead.
4130    SwapInProgress {
4131        module_id: String,
4132    },
4133    /// A swap was refused before anything was spawned.
4134    SwapRefused {
4135        module_id: String,
4136        reason: SwapRefusal,
4137    },
4138    /// A swap spawned a candidate and gave up on it. The candidate has been
4139    /// killed and its slot freed; the incumbent was left serving and was never
4140    /// drained, except in the one `CutoverLost` case described on that arm.
4141    SwapFailed {
4142        module_id: String,
4143        arm: SwapFailureArm,
4144        detail: String,
4145        /// How the candidate exited, when it exited on its own before the
4146        /// supervisor gave up on it.
4147        candidate_exit: Option<ExitReport>,
4148    },
4149}
4150
4151/// Why a swap was refused before a candidate was spawned.
4152#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4153pub enum SwapRefusal {
4154    /// The module's config does not declare `overlap: "safe"`.
4155    OverlapExclusive,
4156    /// The module is not registered, so there is no incumbent to keep serving
4157    /// and nothing a swap would improve on; a plain restart is the tool.
4158    NotRegistered,
4159    /// The module does not speak the subc wire, so a candidate could never
4160    /// register or declare itself ready.
4161    ProtocolNone,
4162    /// The supervisor lacks the forwarding table (to cut routes over) or the
4163    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4164    NotConfigured,
4165    /// A swap is already open for this module.
4166    AlreadySwapping,
4167}
4168
4169impl SwapRefusal {
4170    pub fn as_str(self) -> &'static str {
4171        match self {
4172            Self::OverlapExclusive => "overlap_exclusive",
4173            Self::NotRegistered => "not_registered",
4174            Self::ProtocolNone => "protocol_none",
4175            Self::NotConfigured => "not_configured",
4176            Self::AlreadySwapping => "already_swapping",
4177        }
4178    }
4179}
4180
4181/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4182/// serving and undrained; see `CutoverLost`.
4183#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4184pub enum SwapFailureArm {
4185    /// The candidate process could not be started.
4186    SpawnFailed,
4187    /// The candidate did not register within the readiness budget.
4188    NeverRegistered,
4189    /// The candidate registered but did not declare itself ready in time.
4190    NeverReady,
4191    /// The candidate exited before cutover.
4192    CandidateExited,
4193    /// The candidate declared itself ready but failed its health probe.
4194    CandidateUnhealthy,
4195    /// An operator stop, disable or retire arrived while the candidate warmed.
4196    /// The candidate was killed and the operator's command then carried out on
4197    /// the incumbent.
4198    Interrupted,
4199    /// The candidate's connection closed at the moment of cutover. If it
4200    /// closed before forwarding moved, the incumbent is untouched. If it closed
4201    /// between the forwarding and registry halves of cutover, forwarding can no
4202    /// longer route to the incumbent, so the module is restarted plainly.
4203    CutoverLost,
4204}
4205
4206impl SwapFailureArm {
4207    pub fn as_str(self) -> &'static str {
4208        match self {
4209            Self::SpawnFailed => "spawn_failed",
4210            Self::NeverRegistered => "never_registered",
4211            Self::NeverReady => "never_ready",
4212            Self::CandidateExited => "candidate_exited",
4213            Self::CandidateUnhealthy => "candidate_unhealthy",
4214            Self::Interrupted => "interrupted",
4215            Self::CutoverLost => "cutover_lost",
4216        }
4217    }
4218}
4219
4220impl fmt::Display for SuperviseError {
4221    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4222        match self {
4223            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4224            Self::Spawn {
4225                program,
4226                source,
4227                cgroup_path: Some(cgroup_path),
4228            } => write!(
4229                f,
4230                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4231                cgroup_path.display(),
4232                program.display()
4233            ),
4234            Self::Spawn {
4235                program,
4236                source,
4237                cgroup_path: None,
4238            } => write!(
4239                f,
4240                "failed to spawn module '{}': {source}",
4241                program.display()
4242            ),
4243            Self::Cgroup { module_id, source } => {
4244                write!(
4245                    f,
4246                    "failed to prepare cgroup for module '{module_id}': {source}"
4247                )
4248            }
4249            Self::LaunchNonce { reason } => {
4250                write!(
4251                    f,
4252                    "failed to generate reserved-module launch nonce: {reason}"
4253                )
4254            }
4255            Self::Wait { module_id, source } => {
4256                write!(f, "failed to wait for module '{module_id}': {source}")
4257            }
4258            Self::Kill { module_id, source } => {
4259                write!(f, "failed to kill module '{module_id}': {source}")
4260            }
4261            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4262            Self::Registry(err) => write!(f, "registry error: {err}"),
4263            Self::ReloadUnavailable { module_id, reason } => {
4264                write!(f, "reload unavailable for module '{module_id}': {reason}")
4265            }
4266            Self::Disabled { module_id } => {
4267                write!(
4268                    f,
4269                    "module '{module_id}' is disabled; enable it before restart or reload"
4270                )
4271            }
4272            Self::ReloadFailed { module_id, reason } => {
4273                write!(f, "reload failed for module '{module_id}': {reason}")
4274            }
4275            Self::RegistrationStillActive { module_id, waited } => write!(
4276                f,
4277                "module '{module_id}' registration remained active after waiting {waited:?}"
4278            ),
4279            Self::StatePoisoned { module_id } => match module_id {
4280                Some(module_id) => {
4281                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4282                }
4283                None => write!(f, "supervisor state was poisoned"),
4284            },
4285            Self::CommandClosed { module_id } => {
4286                write!(
4287                    f,
4288                    "supervisor command channel for module '{module_id}' is closed"
4289                )
4290            }
4291            Self::SwapInProgress { module_id } => write!(
4292                f,
4293                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4294            ),
4295            Self::SwapRefused { module_id, reason } => match reason {
4296                SwapRefusal::OverlapExclusive => write!(
4297                    f,
4298                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4299                ),
4300                SwapRefusal::NotRegistered => write!(
4301                    f,
4302                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4303                ),
4304                SwapRefusal::ProtocolNone => write!(
4305                    f,
4306                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4307                ),
4308                SwapRefusal::NotConfigured => write!(
4309                    f,
4310                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4311                ),
4312                SwapRefusal::AlreadySwapping => {
4313                    write!(f, "module '{module_id}' is already being swapped")
4314                }
4315            },
4316            Self::SwapFailed {
4317                module_id,
4318                arm,
4319                detail,
4320                ..
4321            } => write!(
4322                f,
4323                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4324                arm.as_str()
4325            ),
4326        }
4327    }
4328}
4329
4330impl Error for SuperviseError {
4331    fn source(&self) -> Option<&(dyn Error + 'static)> {
4332        match self {
4333            Self::Spawn { source, .. }
4334            | Self::Cgroup { source, .. }
4335            | Self::Wait { source, .. }
4336            | Self::Kill { source, .. } => Some(source),
4337            Self::Forwarding(err) => Some(err),
4338            Self::Registry(err) => Some(err),
4339            Self::LaunchNonce { .. }
4340            | Self::InvalidSpec { .. }
4341            | Self::ReloadUnavailable { .. }
4342            | Self::Disabled { .. }
4343            | Self::ReloadFailed { .. }
4344            | Self::RegistrationStillActive { .. }
4345            | Self::StatePoisoned { .. }
4346            | Self::CommandClosed { .. }
4347            | Self::SwapInProgress { .. }
4348            | Self::SwapRefused { .. }
4349            | Self::SwapFailed { .. } => None,
4350        }
4351    }
4352}
4353
4354pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4355    if spec.module_id.trim().is_empty() {
4356        return Err(SuperviseError::InvalidSpec {
4357            reason: "module_id must not be empty".to_string(),
4358        });
4359    }
4360
4361    Ok(())
4362}
4363
4364#[derive(Debug, Default)]
4365struct HealthProbeRuntime {
4366    configured_health: Option<HealthConfig>,
4367    registered_connection: Option<crate::ConnectionId>,
4368    advertised: bool,
4369    next_probe_at: Option<Instant>,
4370    probe_index: u64,
4371}
4372
4373fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4374    lock_snapshot(snapshot)
4375        .ok()
4376        .and_then(|state| state.spawned_protocol)
4377        .unwrap_or(spec.protocol)
4378}
4379
4380impl HealthProbeRuntime {
4381    fn refresh_registration(
4382        &mut self,
4383        spec: &ModuleSpec,
4384        runtime: &SupervisorRuntimeConfig,
4385        registry: &Registry,
4386        snapshot: &SharedSnapshot,
4387    ) {
4388        if self.configured_health.as_ref() != Some(&runtime.health) {
4389            self.configured_health = Some(runtime.health.clone());
4390            self.next_probe_at = None;
4391            self.registered_connection = None;
4392            self.probe_index = 0;
4393        }
4394        // A non-wire process never registers. Only an explicitly configured
4395        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4396        // health failure for that kind of process.
4397        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4398            self.registered_connection = None;
4399            self.advertised = runtime.health.http.is_some();
4400            if !self.advertised {
4401                self.next_probe_at = None;
4402                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4403                    let unknown = ModuleHealthStatus::default();
4404                    if state.health != unknown {
4405                        state.health = unknown;
4406                    }
4407                });
4408            } else if self.next_probe_at.is_none() {
4409                self.next_probe_at = Some(
4410                    Instant::now()
4411                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4412                );
4413            }
4414            return;
4415        }
4416
4417        let registration = match registry.get_module(&spec.module_id) {
4418            Ok(registration) => registration,
4419            Err(err) => {
4420                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4421                self.advertised = false;
4422                self.next_probe_at = None;
4423                return;
4424            }
4425        };
4426
4427        let Some(registration) = registration else {
4428            self.registered_connection = None;
4429            self.advertised = false;
4430            self.next_probe_at = None;
4431            return;
4432        };
4433
4434        let advertised = registration
4435            .control_ops
4436            .iter()
4437            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4438        if !advertised {
4439            self.registered_connection = Some(registration.connection_id);
4440            self.advertised = false;
4441            self.next_probe_at = None;
4442            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4443                if state.health.status != SupervisorHealthStatus::Unknown
4444                    || state.health.consecutive_failures != 0
4445                    || state.health.last_probe_ms.is_some()
4446                    || state.health.detail.is_some()
4447                    || state.health.metrics.is_some()
4448                {
4449                    state.health.status = SupervisorHealthStatus::Unknown;
4450                    state.health.consecutive_failures = 0;
4451                    state.health.last_probe_ms = None;
4452                    state.health.detail = None;
4453                    state.health.metrics = None;
4454                }
4455            });
4456            return;
4457        }
4458
4459        let reregistered = self.registered_connection != Some(registration.connection_id);
4460        self.registered_connection = Some(registration.connection_id);
4461        self.advertised = true;
4462        if reregistered || self.next_probe_at.is_none() {
4463            self.probe_index = 0;
4464            self.next_probe_at = Some(
4465                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4466            );
4467            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4468                state.health.status = SupervisorHealthStatus::Unknown;
4469                state.health.consecutive_failures = 0;
4470                state.health.detail = None;
4471                state.health.metrics = None;
4472            });
4473        }
4474    }
4475
4476    fn wake_after(&self) -> Option<Duration> {
4477        if !self.advertised {
4478            return None;
4479        }
4480        self.next_probe_at
4481            .map(|next| next.saturating_duration_since(Instant::now()))
4482    }
4483
4484    fn due(&self) -> bool {
4485        self.advertised
4486            && self
4487                .next_probe_at
4488                .is_some_and(|next| Instant::now() >= next)
4489    }
4490
4491    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4492        self.probe_index = self.probe_index.wrapping_add(1);
4493        self.next_probe_at = Some(
4494            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4495        );
4496    }
4497}
4498
4499/// What a failed health probe actually OBSERVED, kept apart from how it reads.
4500///
4501/// This was a struct with a single `message: String`, and every one of the
4502/// fifteen construction sites collapsed into it. Each site knows exactly what it
4503/// saw -- the lane is gone, the module did not answer in time, the module
4504/// answered with the wrong thing -- and `handle_health_probe_failure` then
4505/// treated all of them identically: increment a counter, compare to a threshold,
4506/// restart the module. THE DISTINCTION EXISTED AT EVERY CALL SITE AND WAS
4507/// DESTROYED BEFORE THE DECISION THAT NEEDED IT.
4508///
4509/// The distinction that matters is not severity, it is EVIDENTIAL WEIGHT:
4510///
4511/// * `LaneDead` is PROOF. The module's control connection is gone; nothing will
4512///   answer on it again.
4513/// * `NoAnswer` is ABSENCE OF EVIDENCE. It is consistent with a wedged module
4514///   AND with a perfectly healthy one that lost a CPU race -- which is what
4515///   happens under machine load, and is how this supervisor killed a healthy
4516///   module three times in one day.
4517/// * `BadAnswer` proves the module is ALIVE. It replied; the reply was wrong.
4518///   Restarting on it is defensible, but it is not the silence case and should
4519///   never be counted as one.
4520/// * `Misconfigured` is a daemon-side fault. The module has not been asked
4521///   anything, so it cannot be evidence about the module at all.
4522///
4523/// The asymmetry is the whole point: under saturation the WEAKEST signal is the
4524/// one that fires most often, and while every variant collapsed into one string
4525/// it carried the same weight as the strongest.
4526///
4527/// LIVE BEHAVIOUR TODAY, stated here because this doc block describes the
4528/// DESIGN and a reader stopping at it gets the build backwards: the restart
4529/// decision does NOT yet consult this classification -- consecutive `NoAnswer`
4530/// probes still increment the failure streak and drive escalation at the
4531/// threshold (see `is_proof_of_death` below for why that is deliberate and
4532/// what gates the change). Absence of evidence restarts modules today.
4533#[derive(Debug)]
4534enum HealthProbeEvidence {
4535    /// The module's control lane is gone. Proof of death.
4536    LaneDead,
4537    /// No reply within the deadline. Proves nothing about the module's state.
4538    NoAnswer,
4539    /// The module replied, but not with a usable health report. Proves it is alive.
4540    BadAnswer,
4541    /// The daemon could not ask. Says nothing about the module.
4542    Misconfigured,
4543}
4544
4545#[derive(Debug)]
4546struct HealthProbeError {
4547    evidence: HealthProbeEvidence,
4548    message: String,
4549}
4550
4551impl HealthProbeError {
4552    fn lane_dead(message: impl Into<String>) -> Self {
4553        Self::with(HealthProbeEvidence::LaneDead, message)
4554    }
4555
4556    fn no_answer(message: impl Into<String>) -> Self {
4557        Self::with(HealthProbeEvidence::NoAnswer, message)
4558    }
4559
4560    fn bad_answer(message: impl Into<String>) -> Self {
4561        Self::with(HealthProbeEvidence::BadAnswer, message)
4562    }
4563
4564    fn misconfigured(message: impl Into<String>) -> Self {
4565        Self::with(HealthProbeEvidence::Misconfigured, message)
4566    }
4567
4568    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4569        Self {
4570            evidence,
4571            message: message.into(),
4572        }
4573    }
4574
4575    /// Whether this observation is proof the module cannot serve.
4576    ///
4577    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4578    /// variant that fires under CPU starvation, and treating it as proof is the
4579    /// defect this enum exists to make impossible to reintroduce silently.
4580    ///
4581    /// NOT YET CONSULTED BY THE RESTART DECISION, deliberately. Requiring proof
4582    /// to restart also needs a bound for the case it excludes -- a genuinely
4583    /// wedged module, alive but never answering -- and that bound must come from
4584    /// the distribution of real late-answer latencies, which nothing measures
4585    /// yet. Landing the classification first makes the later change a one-line
4586    /// decision against evidence that already exists, rather than two unproven
4587    /// changes at once.
4588    #[allow(dead_code)]
4589    fn is_proof_of_death(&self) -> bool {
4590        matches!(self.evidence, HealthProbeEvidence::LaneDead)
4591    }
4592
4593    /// Short stable label for logs and the health snapshot.
4594    ///
4595    /// An operator reading `ck health` currently cannot tell "the module is gone"
4596    /// from "the module did not answer in five seconds", because both render as
4597    /// prose in the same field. These labels are what make the two
4598    /// distinguishable at a glance, and they are what a later restart-policy
4599    /// change will be argued from.
4600    fn label(&self) -> &'static str {
4601        match self.evidence {
4602            HealthProbeEvidence::LaneDead => "lane-dead",
4603            HealthProbeEvidence::NoAnswer => "no-answer",
4604            HealthProbeEvidence::BadAnswer => "bad-answer",
4605            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4606        }
4607    }
4608}
4609
4610impl fmt::Display for HealthProbeError {
4611    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4612        f.write_str(&self.message)
4613    }
4614}
4615
4616async fn run_health_probe_cycle(
4617    spec: &ModuleSpec,
4618    runtime: &SupervisorRuntimeConfig,
4619    registry: &Registry,
4620    process_liveness: &SupervisorProcessLiveness,
4621    snapshot: &SharedSnapshot,
4622    child: &mut Option<SupervisedChild>,
4623) {
4624    let now_ms = unix_ms_now();
4625    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4626        .then_some(runtime.health.http.as_deref())
4627        .flatten();
4628    let result = match http {
4629        Some(url) => probe_http_health(url, runtime.health.deadline).await,
4630        None => probe_module_health(&spec.module_id, runtime, None).await,
4631    };
4632    match result {
4633        Ok(report) => {
4634            handle_health_report(
4635                spec,
4636                runtime,
4637                registry,
4638                process_liveness,
4639                snapshot,
4640                child,
4641                report,
4642                now_ms,
4643            )
4644            .await;
4645        }
4646        Err(err) => {
4647            if http.is_some() {
4648                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4649                    state.health.status = SupervisorHealthStatus::Failing;
4650                });
4651            }
4652            handle_health_probe_failure(
4653                spec,
4654                runtime,
4655                registry,
4656                process_liveness,
4657                snapshot,
4658                child,
4659                err,
4660                now_ms,
4661            )
4662            .await;
4663        }
4664    }
4665}
4666
4667pub(crate) struct HttpProbeTarget<'a> {
4668    address: std::net::SocketAddr,
4669    localhost: bool,
4670    authority: &'a str,
4671    path: String,
4672}
4673
4674/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
4675/// or TLS. A URL cannot turn a local health check into an outbound connection.
4676pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4677    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4678        return Err("must not contain whitespace, controls, or a fragment".into());
4679    }
4680    let rest = url
4681        .strip_prefix("http://")
4682        .ok_or("must use plain http://")?;
4683    let split = rest.find(['/', '?']).unwrap_or(rest.len());
4684    let (authority, suffix) = rest.split_at(split);
4685    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4686        ("::1", rest)
4687    } else {
4688        let split = authority.find(':').unwrap_or(authority.len());
4689        authority.split_at(split)
4690    };
4691    let ip: std::net::IpAddr = match host {
4692        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4693        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4694        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4695    };
4696    let port = if port.is_empty() {
4697        80
4698    } else {
4699        port.strip_prefix(':')
4700            .and_then(|p| p.parse::<u16>().ok())
4701            .filter(|p| *p > 0)
4702            .ok_or("must have a valid nonzero TCP port")?
4703    };
4704    let path = if suffix.is_empty() {
4705        "/".into()
4706    } else if suffix.starts_with('?') {
4707        format!("/{suffix}")
4708    } else {
4709        suffix.into()
4710    };
4711    Ok(HttpProbeTarget {
4712        address: std::net::SocketAddr::new(ip, port),
4713        localhost: host == "localhost",
4714        authority,
4715        path,
4716    })
4717}
4718
4719async fn probe_http_health(
4720    url: &str,
4721    deadline: Duration,
4722) -> Result<HealthReport, HealthProbeError> {
4723    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4724    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4725    // Keep partial diagnostics outside the timed future so cancellation does
4726    // not discard a status line or body bytes already received.
4727    let mut response_status = String::new();
4728    let mut body = Vec::new();
4729    let probe = async {
4730        // Resolve localhost ourselves so a hosts-file override cannot turn
4731        // this into an outbound request, while IPv6-only local servers work.
4732        let connection = match tokio::net::TcpStream::connect(target.address).await {
4733            Err(_) if target.localhost => {
4734                tokio::net::TcpStream::connect((
4735                    std::net::Ipv6Addr::LOCALHOST,
4736                    target.address.port(),
4737                ))
4738                .await
4739            }
4740            result => result,
4741        };
4742        let mut stream = connection.map_err(|error| {
4743            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4744        })?;
4745        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4746            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4747        let mut reader = BufReader::new(stream);
4748        let mut budget = 16 * 1024;
4749        let status = http_line(&mut reader, &mut budget).await?;
4750        let mut words = status.split_ascii_whitespace();
4751        let version = words.next();
4752        let code = words
4753            .next()
4754            .filter(|word| word.len() == 3)
4755            .and_then(|word| word.parse::<u16>().ok());
4756        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4757            || !code.is_some_and(|code| (100..600).contains(&code))
4758        {
4759            return Err(HealthProbeError::bad_answer(format!(
4760                "invalid HTTP status: {status}"
4761            )));
4762        }
4763        let code = code.expect("validated status code");
4764        response_status = status.clone();
4765        let mut length = None;
4766        let mut chunked = false;
4767        loop {
4768            let line = http_line(&mut reader, &mut budget).await?;
4769            if line.is_empty() {
4770                break;
4771            }
4772            if let Some((name, value)) = line.split_once(':') {
4773                if name.eq_ignore_ascii_case("content-length") {
4774                    length = Some(value.trim().parse::<u64>().map_err(|_| {
4775                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
4776                    })?);
4777                } else if name.eq_ignore_ascii_case("transfer-encoding") {
4778                    chunked = value.trim().eq_ignore_ascii_case("chunked");
4779                }
4780            }
4781        }
4782        if chunked {
4783            while body.len() < 200 {
4784                let line = http_line(&mut reader, &mut budget).await?;
4785                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4786                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4787                if size == 0 {
4788                    break;
4789                }
4790                let count = size.min((200 - body.len()) as u64) as usize;
4791                let start = body.len();
4792                (&mut reader)
4793                    .take(count as u64)
4794                    .read_to_end(&mut body)
4795                    .await
4796                    .map_err(|error| {
4797                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4798                    })?;
4799                if body.len() - start != count {
4800                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4801                }
4802                if size > count as u64 || body.len() == 200 {
4803                    break;
4804                }
4805                if !http_line(&mut reader, &mut budget).await?.is_empty() {
4806                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4807                }
4808            }
4809        } else {
4810            reader
4811                .take(length.unwrap_or(200).min(200))
4812                .read_to_end(&mut body)
4813                .await
4814                .map_err(|error| {
4815                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4816                })?;
4817        }
4818        if (200..300).contains(&code) {
4819            Ok(HealthReport::ok())
4820        } else {
4821            Err(HealthProbeError::bad_answer(
4822                "HTTP health endpoint returned non-2xx",
4823            ))
4824        }
4825    };
4826    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4827        Err(HealthProbeError::no_answer(format!(
4828            "HTTP probe timed out after {deadline:?}"
4829        )))
4830    });
4831    if let Err(error) = &mut result {
4832        if !response_status.is_empty() {
4833            error.message = format!(
4834                "{}; {response_status}: {}",
4835                error.message,
4836                String::from_utf8_lossy(&body)
4837            );
4838        }
4839    }
4840    result
4841}
4842
4843async fn http_line(
4844    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4845    remaining: &mut usize,
4846) -> Result<String, HealthProbeError> {
4847    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4848    let mut line = Vec::new();
4849    (&mut *reader)
4850        .take(*remaining as u64)
4851        .read_until(b'\n', &mut line)
4852        .await
4853        .map_err(|error| {
4854            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4855        })?;
4856    *remaining -= line.len();
4857    if !line.ends_with(b"\r\n") {
4858        return Err(HealthProbeError::bad_answer(
4859            "HTTP headers are incomplete or exceed 16 KiB",
4860        ));
4861    }
4862    line.truncate(line.len() - 2);
4863    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4864}
4865
4866async fn probe_module_health(
4867    module_id: &str,
4868    runtime: &SupervisorRuntimeConfig,
4869    drain_deadline: Option<Instant>,
4870) -> Result<HealthReport, HealthProbeError> {
4871    let Some(forwarding) = runtime.forwarding.as_ref() else {
4872        return Err(HealthProbeError::misconfigured(
4873            "supervisor was not configured with a forwarding table",
4874        ));
4875    };
4876    let probe_started_at = Instant::now();
4877    let mut deadline = probe_started_at + runtime.health.deadline;
4878    if let Some(drain_deadline) = drain_deadline {
4879        deadline = deadline.min(drain_deadline);
4880    }
4881    let pending = if drain_deadline.is_some() {
4882        forwarding.begin_drain_health_probe_rpc_for(
4883            module_id,
4884            MODULE_CONTROL_OP_HEALTH_CHECK,
4885            probe_started_at,
4886            deadline,
4887        )
4888    } else {
4889        forwarding.begin_health_probe_rpc_for(
4890            module_id,
4891            MODULE_CONTROL_OP_HEALTH_CHECK,
4892            probe_started_at,
4893            deadline,
4894        )
4895    }
4896    .map_err(|err| {
4897        // The endpoint is not registered, so there is no live control lane to
4898        // ask. That is the module being absent, not slow.
4899        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4900    })?;
4901    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4902}
4903
4904/// [`probe_module_health`] for one endpoint rather than the id's active one.
4905///
4906/// A swap probes two processes that no by-id lookup reaches: its candidate
4907/// before cutover, and its superseded incumbent (for busy gauges) while the
4908/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
4909/// bounds the by-id drain probe.
4910async fn probe_endpoint_health(
4911    endpoint: crate::ModuleEndpointId,
4912    runtime: &SupervisorRuntimeConfig,
4913    deadline_cap: Option<Instant>,
4914) -> Result<HealthReport, HealthProbeError> {
4915    let Some(forwarding) = runtime.forwarding.as_ref() else {
4916        return Err(HealthProbeError::misconfigured(
4917            "supervisor was not configured with a forwarding table",
4918        ));
4919    };
4920    let probe_started_at = Instant::now();
4921    let mut deadline = probe_started_at + runtime.health.deadline;
4922    if let Some(cap) = deadline_cap {
4923        deadline = deadline.min(cap);
4924    }
4925    let pending = forwarding
4926        .begin_endpoint_health_probe_rpc_for(
4927            endpoint,
4928            MODULE_CONTROL_OP_HEALTH_CHECK,
4929            probe_started_at,
4930            deadline,
4931        )
4932        .map_err(|err| {
4933            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4934        })?;
4935    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4936}
4937
4938/// Send a begun health probe and classify its answer.
4939async fn await_health_probe(
4940    forwarding: &ForwardingTable,
4941    pending: PendingModuleControlRpc,
4942    deadline: Instant,
4943    probe_budget: Duration,
4944) -> Result<HealthReport, HealthProbeError> {
4945    let PendingModuleControlRpc {
4946        endpoint,
4947        module_sink,
4948        negotiated_ver,
4949        corr,
4950        receiver,
4951    } = pending;
4952    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4953        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
4954    })?;
4955    let frame = Frame::build_with_version(
4956        negotiated_ver,
4957        FrameType::Request,
4958        control_flags(),
4959        0,
4960        0,
4961        corr,
4962        body,
4963    )
4964    .map_err(|err| {
4965        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
4966    })?;
4967
4968    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
4969    // blocks waiting for capacity when the module's egress queue is full, and an
4970    // unbounded await here freezes the whole supervision actor (it stops polling
4971    // Child::wait and supervisor commands), making the module unrecoverable
4972    // in-band. On timeout the probe fails like any transport failure.
4973    match timeout_at(deadline, module_sink.send(frame)).await {
4974        Ok(Ok(())) => {}
4975        Ok(Err(err)) => {
4976            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
4977            // A closed sink means the module's egress channel is gone -- the
4978            // receiving half is dropped when its connection tears down. Proof.
4979            return Err(HealthProbeError::lane_dead(format!(
4980                "failed to send health.check: {err}"
4981            )));
4982        }
4983        Err(_elapsed) => {
4984            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
4985            // A full egress queue means the module is not draining its socket, which
4986            // is consistent with a wedged module AND with one whose reader is merely
4987            // starved. Silence, not proof.
4988            return Err(HealthProbeError::no_answer(
4989                "health.check send timed out before enqueue (module egress full)",
4990            ));
4991        }
4992    }
4993
4994    match timeout_at(deadline, receiver).await {
4995        // Each arm records WHAT WAS OBSERVED. Four of them are the module
4996        // demonstrably answering -- rejected, non-health, malformed, wrong op --
4997        // and those prove it is alive even though the probe failed.
4998        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4999            response.health_report().ok_or_else(|| {
5000                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5001            })
5002        }
5003        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5004            format!("health.check rejected: {}", body.message),
5005        )),
5006        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5007            Err(HealthProbeError::lane_dead(message))
5008        }
5009        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5010            Err(HealthProbeError::bad_answer(message))
5011        }
5012        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5013            Err(HealthProbeError::bad_answer(format!(
5014                "expected module-control op '{expected}', got '{actual}'"
5015            )))
5016        }
5017        // A reply that crosses the deadline before this waiter observes it is
5018        // still proof of life. The forwarding path records its end-to-end latency
5019        // before delivering this classification.
5020        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5021            "module answered health.check after its daemon deadline",
5022        )),
5023        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5024            "health.check waiter was canceled before the module responded",
5025        )),
5026        Err(_) => {
5027            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5028            Err(HealthProbeError::no_answer(format!(
5029                "module did not answer health.check within {probe_budget:?}"
5030            )))
5031        }
5032    }
5033}
5034
5035#[allow(clippy::too_many_arguments)]
5036async fn handle_health_report(
5037    spec: &ModuleSpec,
5038    runtime: &SupervisorRuntimeConfig,
5039    registry: &Registry,
5040    process_liveness: &SupervisorProcessLiveness,
5041    snapshot: &SharedSnapshot,
5042    child: &mut Option<SupervisedChild>,
5043    report: HealthReport,
5044    now_ms: u64,
5045) {
5046    let status = supervisor_health_status(report.status);
5047    let detail = report.detail.clone();
5048    let metrics = truncate_health_metrics(report.metrics);
5049    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5050        state.health.status = status;
5051        state.health.last_probe_ms = Some(now_ms);
5052        state.health.detail = detail.clone();
5053        state.health.metrics = metrics.clone();
5054        state.health.consecutive_failures = 0;
5055    });
5056
5057    let action = match report.status {
5058        HealthStatus::Ok => return,
5059        HealthStatus::Degraded => runtime.health.on_degraded,
5060        HealthStatus::Failing => runtime.health.on_failing,
5061    };
5062    apply_l3_health_action(
5063        spec,
5064        runtime,
5065        registry,
5066        process_liveness,
5067        snapshot,
5068        child,
5069        status,
5070        detail.as_deref(),
5071        action,
5072        now_ms,
5073    )
5074    .await;
5075}
5076
5077#[allow(clippy::too_many_arguments)]
5078async fn handle_health_probe_failure(
5079    spec: &ModuleSpec,
5080    runtime: &SupervisorRuntimeConfig,
5081    registry: &Registry,
5082    process_liveness: &SupervisorProcessLiveness,
5083    snapshot: &SharedSnapshot,
5084    child: &mut Option<SupervisedChild>,
5085    err: HealthProbeError,
5086    now_ms: u64,
5087) {
5088    let threshold = runtime.health.failure_threshold.max(1);
5089    let mut failures = 0;
5090    // Carry the evidence class into the operator-visible detail. Without it,
5091    // "module did not answer within 5s" and "the control lane is gone" are two
5092    // prose strings in the same field, and the reader has to know the codebase to
5093    // tell which one is proof of anything.
5094    let detail = format!("[{}] {err}", err.label());
5095    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5096        // A failed wire probe invalidates the last report, even before the
5097        // restart threshold. HTTP probes already mark failures as Failing.
5098        if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5099            state.health.status = SupervisorHealthStatus::Unknown;
5100        }
5101        state.health.last_probe_ms = Some(now_ms);
5102        state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5103        state.health.detail = Some(detail.clone());
5104        state.health.metrics = None;
5105        failures = state.health.consecutive_failures;
5106    });
5107
5108    if failures < threshold {
5109        warn!(
5110            module_id = %spec.module_id,
5111            consecutive_failures = failures,
5112            threshold,
5113            evidence = err.label(),
5114            detail = %detail,
5115            "health.check probe failed"
5116        );
5117        return;
5118    }
5119
5120    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5121        state.state = ModuleState::Unresponsive;
5122        state.health.status = SupervisorHealthStatus::Unresponsive;
5123    });
5124    // The evidence class is logged at the kill site because this is the line an
5125    // operator reads after an unexplained restart. A streak of `no-answer` under
5126    // machine load is the known false-positive shape; a `lane-dead` is not.
5127    if runtime.health.critical {
5128        error!(
5129            module_id = %spec.module_id,
5130            status = "unresponsive",
5131            evidence = err.label(),
5132            detail = %detail,
5133            "critical module health alert"
5134        );
5135    } else {
5136        warn!(
5137            module_id = %spec.module_id,
5138            status = "unresponsive",
5139            evidence = err.label(),
5140            detail = %detail,
5141            "module health threshold breached"
5142        );
5143    }
5144    if let Err(err) = health_restart_child(
5145        spec,
5146        runtime,
5147        registry,
5148        process_liveness,
5149        snapshot,
5150        child,
5151        SupervisorHealthStatus::Unresponsive,
5152        Some(&detail),
5153        now_ms,
5154    )
5155    .await
5156    {
5157        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5158    }
5159}
5160
5161#[allow(clippy::too_many_arguments)]
5162async fn apply_l3_health_action(
5163    spec: &ModuleSpec,
5164    runtime: &SupervisorRuntimeConfig,
5165    registry: &Registry,
5166    process_liveness: &SupervisorProcessLiveness,
5167    snapshot: &SharedSnapshot,
5168    child: &mut Option<SupervisedChild>,
5169    status: SupervisorHealthStatus,
5170    detail: Option<&str>,
5171    action: HealthAction,
5172    now_ms: u64,
5173) {
5174    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5175    match action {
5176        HealthAction::Report => {
5177            info!(
5178                module_id = %spec.module_id,
5179                status = ?status,
5180                detail,
5181                "module reported non-ok health"
5182            );
5183        }
5184        HealthAction::Alert => {
5185            error!(
5186                module_id = %spec.module_id,
5187                status = ?status,
5188                detail,
5189                "module health alert"
5190            );
5191        }
5192        HealthAction::Restart => {
5193            if let Err(err) = health_restart_child(
5194                spec,
5195                runtime,
5196                registry,
5197                process_liveness,
5198                snapshot,
5199                child,
5200                status,
5201                detail,
5202                now_ms,
5203            )
5204            .await
5205            {
5206                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5207            }
5208        }
5209    }
5210}
5211
5212#[allow(clippy::too_many_arguments)]
5213async fn health_restart_child(
5214    spec: &ModuleSpec,
5215    runtime: &SupervisorRuntimeConfig,
5216    registry: &Registry,
5217    process_liveness: &SupervisorProcessLiveness,
5218    snapshot: &SharedSnapshot,
5219    child: &mut Option<SupervisedChild>,
5220    status: SupervisorHealthStatus,
5221    detail: Option<&str>,
5222    now_ms: u64,
5223) -> Result<(), SuperviseError> {
5224    let (enabled, schedule) = {
5225        let mut state = lock_snapshot(snapshot)?;
5226        let enabled = state.enabled;
5227        let schedule = if enabled {
5228            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5229        } else {
5230            None
5231        };
5232        (enabled, schedule)
5233    };
5234
5235    if !enabled {
5236        return Err(SuperviseError::Disabled {
5237            module_id: spec.module_id.clone(),
5238        });
5239    }
5240
5241    if schedule.is_none() {
5242        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5243        error!(
5244            module_id = %spec.module_id,
5245            status = ?status,
5246            detail,
5247            max_restarts = runtime.restart_policy.max_restarts,
5248            window_secs = runtime.restart_policy.window.as_secs(),
5249            reason = %runtime.restart_policy.budget_exhausted_detail(),
5250            "health restart budget exhausted; marking module failed"
5251        );
5252        let stop_notice = begin_forwarding_drain_if_configured(
5253            spec,
5254            runtime,
5255            registry,
5256            snapshot,
5257            Some(true),
5258            RouteCloseReason::Disable,
5259        )
5260        .await?;
5261        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5262            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5263        })?;
5264        drain_optional_child(
5265            &spec.module_id,
5266            spec.protocol,
5267            stop_notice,
5268            registry,
5269            runtime.forwarding.as_deref(),
5270            snapshot,
5271            &runtime.terminal_ring,
5272            &runtime.spawn_events,
5273            child,
5274            runtime.drain_timeout,
5275            ModuleState::Failed,
5276            Some(true),
5277        )
5278        .await?;
5279        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5280        return Ok(());
5281    }
5282
5283    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5284    let mut restart_count = 0;
5285    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5286        restart_count = state.crash_restarts.len();
5287        state.state = ModuleState::Unresponsive;
5288        state.health.status = status;
5289        state.health.last_action = Some(HealthAction::Restart.to_string());
5290        state.health.last_action_ms = Some(now_ms);
5291    })?;
5292    warn!(
5293        module_id = %spec.module_id,
5294        status = ?status,
5295        detail,
5296        restart_count,
5297        restart_in_window = schedule.restart_in_window,
5298        delay_ms = schedule.delay.as_millis() as u64,
5299        "health-triggered module restart"
5300    );
5301
5302    let stop_notice = begin_forwarding_drain_if_configured(
5303        spec,
5304        runtime,
5305        registry,
5306        snapshot,
5307        Some(true),
5308        RouteCloseReason::Restart,
5309    )
5310    .await?;
5311    drain_optional_child(
5312        &spec.module_id,
5313        spec.protocol,
5314        stop_notice,
5315        registry,
5316        runtime.forwarding.as_deref(),
5317        snapshot,
5318        &runtime.terminal_ring,
5319        &runtime.spawn_events,
5320        child,
5321        runtime.drain_timeout,
5322        ModuleState::Restarting,
5323        Some(true),
5324    )
5325    .await?;
5326    schedule_respawn(
5327        runtime,
5328        snapshot,
5329        &spec.module_id,
5330        schedule.delay,
5331        RespawnKind::Spawn,
5332    )
5333}
5334
5335fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5336    if let Some(reply) = runtime
5337        .deferred_reload_reply
5338        .lock()
5339        .unwrap_or_else(|p| p.into_inner())
5340        .take()
5341    {
5342        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5343            module_id: module_id.to_string(),
5344            reason: reason.to_string(),
5345        }));
5346    }
5347}
5348
5349fn schedule_respawn(
5350    runtime: &SupervisorRuntimeConfig,
5351    snapshot: &SharedSnapshot,
5352    module_id: &str,
5353    delay: Duration,
5354    kind: RespawnKind,
5355) -> Result<(), SuperviseError> {
5356    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5357    update_snapshot(snapshot, Some(module_id), |state| {
5358        state.respawn_pending = true
5359    })?;
5360    *runtime
5361        .scheduled_respawn
5362        .lock()
5363        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5364        deadline: Instant::now() + delay,
5365        kind,
5366    });
5367    Ok(())
5368}
5369
5370fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5371    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5372        state.health.last_action = Some(action);
5373        state.health.last_action_ms = Some(now_ms);
5374    });
5375}
5376
5377fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5378    match status {
5379        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5380        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5381        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5382    }
5383}
5384
5385/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5386/// returned to every `supervisor.list` and `supervisor.health` caller.
5387///
5388/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5389/// path: that request exists to return a module's complete metrics object, and
5390/// `ck health <module-id>` documents it as the way to see what the cached view
5391/// truncates. The asymmetry is the feature.
5392///
5393/// So a new caller must decide which side it is on rather than assume the cap is
5394/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5395/// the truncation that path exists to avoid.
5396fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5397    let metrics = metrics?;
5398    match serde_json::to_vec(&metrics) {
5399        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5400            "truncated": true,
5401            "original_bytes": encoded.len(),
5402        })),
5403        Ok(_) | Err(_) => Some(metrics),
5404    }
5405}
5406
5407/// Spread health probes so a fleet-wide restart does not converge them.
5408///
5409/// The delay is derived from the module id and probe index rather than a random
5410/// source, so it is deterministic per module: a module keeps its own offset
5411/// across daemon restarts instead of re-rolling into a collision.
5412fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5413    if cadence.is_zero() {
5414        return Duration::ZERO;
5415    }
5416    let cadence_ms = cadence.as_millis() as u64;
5417    // This early return is REDUNDANT, deliberately, and a mutation run will show
5418    // it surviving removal. Recording why here so the next person to notice does
5419    // not have to re-derive it:
5420    //
5421    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5422    //   a zero cadence and builds the Duration from whole milliseconds, so a
5423    //   sub-millisecond cadence cannot come from config.
5424    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5425    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5426    //   -- exactly what this returns.
5427    //
5428    // Kept as a guard against a future widening of the config parser (accepting
5429    // microseconds, say), which would make the sub-millisecond case reachable.
5430    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5431    // divides by zero. Remove this and nothing changes.
5432    if cadence_ms == 0 {
5433        return cadence;
5434    }
5435    // Note that this never returns less than one cadence, including for the FIRST
5436    // probe. So a freshly registered module reports health `unknown` for a full
5437    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5438    // ready to answer.
5439    //
5440    // That is a property of the supervisor's schedule, not of any module: an
5441    // operator watching a restart sees `unknown` and cannot tell it from a module
5442    // that is slow to warm. Measured on two unrelated modules, both flipping to
5443    // `ok` between 22s and 32s after restart.
5444    //
5445    // Left as-is because spreading the first probe is what keeps a fleet-wide
5446    // restart from firing fourteen simultaneous probes into a cold machine. The
5447    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5448    // that thundering herd for a faster first reading.
5449    let jitter_span = (cadence_ms / 10).max(1);
5450    let hash = module_id.as_bytes().iter().fold(
5451        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5452        |acc, byte| {
5453            acc.wrapping_mul(1099511628211)
5454                .wrapping_add(u64::from(*byte))
5455        },
5456    );
5457    cadence + Duration::from_millis(hash % jitter_span)
5458}
5459
5460#[cfg(test)]
5461mod tests {
5462    use super::*;
5463
5464    #[test]
5465    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5466        let handle = SupervisorHandle::new();
5467        let module_id = "readded-tombstone";
5468        handle.record_rescan_removal(module_id);
5469        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5470
5471        handle.apply_identity_configuration(&ModuleSpec {
5472            module_id: module_id.to_string(),
5473            program: PathBuf::from("/test/module"),
5474            args: Vec::new(),
5475            env: Vec::new(),
5476            reserved: false,
5477            reserved_prefixes: Vec::new(),
5478            protocol: ModuleProtocol::Subc,
5479            overlap: Default::default(),
5480        });
5481
5482        assert!(
5483            handle.removal_tombstone_age_ms(module_id).is_none(),
5484            "a re-added module must not retain a stale removal tombstone"
5485        );
5486    }
5487
5488    /// What one module's owner looked like from the control plane at the
5489    /// instant after its first process was spawned.
5490    #[derive(Debug, PartialEq, Eq)]
5491    struct OwnerInSpawnWindow {
5492        module_id: String,
5493        configured: bool,
5494        on_roster: bool,
5495        admission_refusal: Option<&'static str>,
5496    }
5497
5498    /// A supervised module's process can connect, register, sync its scopes
5499    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5500    /// the module on the roster until afterward; configured supervision does
5501    /// so before spawning. In both cases the owner must
5502    /// already read as configured, so a scoped `route.open` against it is
5503    /// refused as retryable `scope_not_synced` and not as terminal
5504    /// `scope_not_live` ("will never sync").
5505    ///
5506    /// The hook runs just after each initial spawn attempt, before returning to
5507    /// the caller, so no race with a real child's registration is needed.
5508    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5509    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5510        use crate::scopes::ScopeTable;
5511        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5512
5513        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5514        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5515            module_id: module_id.to_string(),
5516            program,
5517            args: Vec::new(),
5518            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5519                .into_iter()
5520                .map(|key| (key.to_string(), dir.path().display().to_string()))
5521                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5522                .collect(),
5523            reserved: false,
5524            reserved_prefixes: Vec::new(),
5525            protocol: ModuleProtocol::Subc,
5526            overlap: Default::default(),
5527        };
5528        let live = super::terminal_history_tests::fake_aft_stub_path();
5529        let missing = dir.path().join("definitely-missing-module");
5530
5531        let handle = SupervisorHandle::new();
5532        let mut supervisor =
5533            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5534                .with_handle(handle.clone());
5535        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5536        let hook_handle = handle.clone();
5537        let hook_observed = Arc::clone(&observed);
5538        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5539            // Exactly what the control plane computes for a scoped route.open
5540            // naming this module as the owner of a scope it has not synced.
5541            let configured = hook_handle.is_configured(module_id);
5542            let selector = ScopeSelector::new(
5543                Principal::Reserved {
5544                    module_id: module_id.to_string(),
5545                },
5546                "s".to_string(),
5547                Some(1),
5548            );
5549            let carrier = Principal::Reserved {
5550                module_id: "carrier".to_string(),
5551            };
5552            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5553                .admit(&carrier, module_id, &selector, configured)
5554            {
5555                Ok(_) => None,
5556                Err(refusal) => Some(refusal.code),
5557            };
5558            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5559                module_id: module_id.to_string(),
5560                configured,
5561                on_roster: hook_handle.get(module_id).is_some(),
5562                admission_refusal,
5563            });
5564        })));
5565
5566        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5567        let configured = supervisor
5568            .supervise_configured(stub("configured", live.clone()), true)
5569            .unwrap();
5570        let with_health = supervisor
5571            .supervise_configured_with_health(
5572                stub("with-health", live.clone()),
5573                true,
5574                HealthConfig::default(),
5575                None,
5576                RestartPolicy::default(),
5577            )
5578            .unwrap();
5579        // The failed-spawn path still puts the module on the roster (as
5580        // failed), so it is configured throughout.
5581        let failed = supervisor
5582            .supervise_configured_with_health(
5583                stub("failed-spawn", missing.clone()),
5584                true,
5585                HealthConfig::default(),
5586                None,
5587                RestartPolicy::default(),
5588            )
5589            .unwrap();
5590        // A failed plain `spawn` puts nothing on the roster, so its mark is
5591        // taken back once the spawn has failed.
5592        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5593
5594        let expected = [
5595            "plain",
5596            "configured",
5597            "with-health",
5598            "failed-spawn",
5599            "spawn-error",
5600        ]
5601        .into_iter()
5602        .map(|module_id| OwnerInSpawnWindow {
5603            module_id: module_id.to_string(),
5604            configured: true,
5605            on_roster: !matches!(module_id, "plain" | "spawn-error"),
5606            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5607        })
5608        .collect::<Vec<_>>();
5609        assert_eq!(*observed.lock().unwrap(), expected);
5610
5611        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5612            assert!(
5613                handle.get(module_id).is_some(),
5614                "{module_id} is on the roster"
5615            );
5616            assert!(
5617                handle.is_configured(module_id),
5618                "{module_id} stays configured"
5619            );
5620        }
5621        assert!(handle.get("spawn-error").is_none());
5622        assert!(
5623            !handle.is_configured("spawn-error"),
5624            "a plain spawn that failed must not leave its module marked configured"
5625        );
5626
5627        // Leaving the roster clears the mark with it.
5628        handle.retire("failed-spawn");
5629        assert!(!handle.is_configured("failed-spawn"));
5630
5631        for module in [plain, configured, with_health] {
5632            module.stop().await.unwrap();
5633        }
5634        drop(failed);
5635    }
5636
5637    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5638        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5639        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5640            snapshot.process_alive = true;
5641            snapshot.pid = Some(41);
5642            snapshot.spawned_at_ms = Some(42);
5643            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5644            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5645                device: 43,
5646                inode: 44,
5647                #[cfg(windows)]
5648                windows_image: None,
5649            });
5650        })
5651        .unwrap();
5652        snapshot
5653    }
5654
5655    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5656        let snapshot = lock_snapshot(snapshot).unwrap();
5657        assert!(!snapshot.process_alive);
5658        assert_eq!(snapshot.pid, None);
5659        assert_eq!(snapshot.spawned_at_ms, None);
5660        assert_eq!(snapshot.spawned_from, None);
5661        assert_eq!(snapshot.spawned_file_identity, None);
5662    }
5663
5664    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5665    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5666        let supervisor =
5667            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5668        let mut runtime = supervisor.runtime_config();
5669        runtime.test_seed_stale_facts_before_enable_spawn = true;
5670        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5671        let mut child = None;
5672        let spec = ModuleSpec {
5673            module_id: "failed-enable-clears-facts".to_string(),
5674            program: PathBuf::from("/definitely/missing/failed-enable-module"),
5675            args: Vec::new(),
5676            env: Vec::new(),
5677            reserved: false,
5678            reserved_prefixes: Vec::new(),
5679            protocol: ModuleProtocol::Subc,
5680            overlap: Default::default(),
5681        };
5682
5683        let result = set_child_enabled(
5684            &spec,
5685            &runtime,
5686            &supervisor.registry,
5687            &supervisor.process_liveness,
5688            &snapshot,
5689            &mut child,
5690            true,
5691        )
5692        .await;
5693
5694        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5695        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5696        assert_snapshot_process_facts_cleared(&snapshot);
5697    }
5698
5699    #[tokio::test]
5700    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5701        let supervisor =
5702            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5703        let runtime = supervisor.runtime_config();
5704        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5705            ModuleState::Restarting,
5706            true,
5707        )));
5708        let spec = ModuleSpec {
5709            module_id: "start-stranded-restarting".to_string(),
5710            program: super::terminal_history_tests::fake_aft_stub_path(),
5711            args: Vec::new(),
5712            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5713            reserved: false,
5714            reserved_prefixes: Vec::new(),
5715            protocol: ModuleProtocol::None,
5716            overlap: Default::default(),
5717        };
5718        let mut child = None;
5719        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5720        assert!(!super::set_child_enabled(
5721            &spec,
5722            &runtime,
5723            &Registry::default(),
5724            &supervisor.process_liveness,
5725            &snapshot,
5726            &mut child,
5727            true
5728        )
5729        .await
5730        .unwrap());
5731        assert!(child.is_none());
5732        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5733        assert!(super::set_child_enabled(
5734            &spec,
5735            &runtime,
5736            &Registry::default(),
5737            &supervisor.process_liveness,
5738            &snapshot,
5739            &mut child,
5740            true
5741        )
5742        .await
5743        .unwrap());
5744        assert_eq!(
5745            lock_snapshot(&snapshot).unwrap().state,
5746            ModuleState::Running
5747        );
5748        let mut child = child.unwrap();
5749        child.start_kill().unwrap();
5750        child.wait().await.unwrap();
5751    }
5752
5753    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5754    async fn failed_reload_spawn_clears_current_process_facts() {
5755        let supervisor =
5756            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5757        let mut runtime = supervisor.runtime_config();
5758        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5759        let snapshot = stale_process_snapshot(ModuleState::Running, true);
5760        let mut child = None;
5761        let spec = ModuleSpec {
5762            module_id: "failed-reload-clears-facts".to_string(),
5763            program: PathBuf::from("/unused/failed-reload-module"),
5764            args: Vec::new(),
5765            env: Vec::new(),
5766            reserved: false,
5767            reserved_prefixes: Vec::new(),
5768            protocol: ModuleProtocol::Subc,
5769            overlap: Default::default(),
5770        };
5771
5772        let result = handle_reload_spawn_failure(
5773            &spec,
5774            &runtime,
5775            &supervisor.process_liveness,
5776            &snapshot,
5777            &mut child,
5778            "forced reload spawn failure".to_string(),
5779        )
5780        .await;
5781
5782        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5783        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5784        assert_snapshot_process_facts_cleared(&snapshot);
5785    }
5786
5787    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5788    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5789        let supervisor =
5790            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5791        let snapshot = stale_process_snapshot(ModuleState::Running, true);
5792        let module = supervisor.supervised_module(
5793            ModuleSpec {
5794                module_id: "drop-clears-facts".to_string(),
5795                program: PathBuf::from("/unused/drop-module"),
5796                args: Vec::new(),
5797                env: Vec::new(),
5798                reserved: false,
5799                reserved_prefixes: Vec::new(),
5800                protocol: ModuleProtocol::Subc,
5801                overlap: Default::default(),
5802            },
5803            supervisor.runtime_config(),
5804            Arc::clone(&snapshot),
5805            None,
5806        );
5807        assert!(!module
5808            .inner
5809            .monitor
5810            .lock()
5811            .unwrap()
5812            .as_ref()
5813            .unwrap()
5814            .is_finished());
5815
5816        drop(module);
5817
5818        assert_eq!(
5819            lock_snapshot(&snapshot).unwrap().state,
5820            ModuleState::Stopped
5821        );
5822        assert_snapshot_process_facts_cleared(&snapshot);
5823    }
5824
5825    #[cfg(unix)]
5826    #[tokio::test]
5827    async fn rescan_preserves_running_protocol_until_respawn() {
5828        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5829        let initial = ModuleSpec {
5830            module_id: "rescan-protocol".into(),
5831            program: PathBuf::from("/bin/sleep"),
5832            args: vec!["60".into()],
5833            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5834                .into_iter()
5835                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
5836                .collect(),
5837            reserved: false,
5838            reserved_prefixes: vec![],
5839            protocol: ModuleProtocol::None,
5840            overlap: Default::default(),
5841        };
5842        let supervisor =
5843            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5844        let module = supervisor.spawn(initial.clone()).unwrap();
5845        assert!(module.status().unwrap().live);
5846        let mut next = initial;
5847        next.protocol = ModuleProtocol::Subc;
5848        module
5849            .update_configuration(next.clone(), HealthConfig::default(), None)
5850            .await
5851            .unwrap();
5852        assert!(
5853            module.status().unwrap().live,
5854            "rescan must not require HELLO from the old non-wire process"
5855        );
5856        let runtime = supervisor.runtime_config();
5857        let action = on_child_exit(
5858            &next,
5859            RestartPolicy::default(),
5860            &supervisor.registry,
5861            &module.inner.snapshot,
5862            &runtime.terminal_ring,
5863            &runtime.spawn_events,
5864            &runtime.child_roster,
5865            ExitReport {
5866                kind: ExitKind::Clean,
5867                code: Some(0),
5868                signal: None,
5869                at_ms: unix_ms_now(),
5870            },
5871        )
5872        .await;
5873        assert!(
5874            matches!(action, NextAction::Restart { .. }),
5875            "the old non-wire process's clean exit must restart"
5876        );
5877        module.drain().await.unwrap();
5878    }
5879
5880    #[cfg(unix)]
5881    #[tokio::test]
5882    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
5883        use std::os::unix::fs::PermissionsExt;
5884        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
5885        let script = dir.join("module.sh");
5886        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
5887        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
5888        let record_path = dir.join("live-children.json");
5889        let supervisor =
5890            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5891                .with_live_children_record(&record_path);
5892        for (program, args) in [
5893            (PathBuf::from("sleep"), vec!["60".into()]),
5894            (script, vec![]),
5895        ] {
5896            let spec = ModuleSpec {
5897                module_id: "image-identity".into(),
5898                program: program.clone(),
5899                args,
5900                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5901                    .into_iter()
5902                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
5903                    .collect(),
5904                reserved: false,
5905                reserved_prefixes: vec![],
5906                protocol: ModuleProtocol::None,
5907                overlap: Default::default(),
5908            };
5909            let module = supervisor.spawn(spec).unwrap();
5910            #[cfg(target_os = "macos")]
5911            {
5912                // SETEXEC confirmation is asynchronous; the orphan record must
5913                // identify the final image, never the intermediate trampoline.
5914                let deadline = Instant::now() + Duration::from_secs(5);
5915                while crate::live_children::read_record(&record_path)
5916                    .unwrap()
5917                    .iter()
5918                    .all(|entry| entry.executable.is_none())
5919                {
5920                    assert!(Instant::now() < deadline, "module image was not confirmed");
5921                    tokio::time::sleep(Duration::from_millis(5)).await;
5922                }
5923            }
5924            let entry = crate::live_children::read_record(&record_path)
5925                .unwrap()
5926                .pop()
5927                .unwrap();
5928            let observed = subc_os::Process::open(entry.pid)
5929                .unwrap()
5930                .unwrap()
5931                .observe()
5932                .unwrap();
5933            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
5934            module.drain().await.unwrap();
5935            assert_eq!(
5936                verdict,
5937                crate::live_children::IdentityVerdict::Matches,
5938                "program {program:?}: recorded {entry:?}, observed {observed:?}"
5939            );
5940        }
5941    }
5942
5943    #[cfg(unix)]
5944    fn http_fixture(
5945        dir: &std::path::Path,
5946        url: &str,
5947        threshold: u32,
5948    ) -> crate::daemon_config::ConfiguredModule {
5949        let path = dir.join("subc.jsonc");
5950        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
5951            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
5952            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
5953            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
5954            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
5955        }}}).to_string()).unwrap();
5956        crate::daemon_config::load(&path)
5957            .unwrap()
5958            .unwrap()
5959            .modules
5960            .pop()
5961            .unwrap()
5962    }
5963
5964    #[cfg(unix)]
5965    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
5966        timeout(Duration::from_secs(5), async {
5967            loop {
5968                if module.status().unwrap().health.status == status {
5969                    break;
5970                }
5971                sleep(Duration::from_millis(5)).await;
5972            }
5973        })
5974        .await
5975        .unwrap_or_else(|_| {
5976            panic!(
5977                "expected {status:?}, got {:?}",
5978                module.status().unwrap().health
5979            )
5980        });
5981    }
5982
5983    #[cfg(unix)]
5984    #[tokio::test]
5985    async fn http_health_status_flips_ok_failing_ok() {
5986        use tokio::io::{AsyncReadExt, AsyncWriteExt};
5987        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
5988        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
5989        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
5990        let serving_status = status.clone();
5991        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
5992        let server = tokio::spawn(async move {
5993            loop {
5994                let (mut stream, _) = listener.accept().await.unwrap();
5995                let mut request = [0u8; 2048];
5996                let count = stream.read(&mut request).await.unwrap();
5997                assert!(count > 0, "a probe must send an HTTP request");
5998                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
5999                let body = if code == 200 {
6000                    "ready"
6001                } else {
6002                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6003                };
6004                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6005                let _ = stream.write_all(response.as_bytes()).await;
6006            }
6007        });
6008        let configured = http_fixture(&dir, &url, 1000);
6009        let module =
6010            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6011                .supervise_configured_with_health(
6012                    configured.module_spec(),
6013                    true,
6014                    configured.health,
6015                    configured.drain_timeout_ms,
6016                    configured.restart,
6017                )
6018                .unwrap();
6019        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6020        status.store(503, std::sync::atomic::Ordering::SeqCst);
6021        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6022        assert!(module
6023            .status()
6024            .unwrap()
6025            .health
6026            .detail
6027            .unwrap()
6028            .contains("scratch failure"));
6029        status.store(200, std::sync::atomic::Ordering::SeqCst);
6030        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6031        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6032        let before = module.status().unwrap();
6033        let (spec, mut health) = module.configuration().unwrap();
6034        health.http = None;
6035        module
6036            .update_configuration(spec.clone(), health.clone(), Some(10))
6037            .await
6038            .unwrap();
6039        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6040        health.http = Some(url);
6041        module
6042            .update_configuration(spec, health, Some(10))
6043            .await
6044            .unwrap();
6045        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6046        assert_eq!(
6047            module.status().unwrap().pid,
6048            before.pid,
6049            "changing a probe must apply live, not restart its process"
6050        );
6051        let (spec, mut health) = module.configuration().unwrap();
6052        health.failure_threshold = 2;
6053        module
6054            .update_configuration(spec, health, Some(10))
6055            .await
6056            .unwrap();
6057        status.store(503, std::sync::atomic::Ordering::SeqCst);
6058        timeout(Duration::from_secs(5), async {
6059            while module.status().unwrap().spawn_generation == before.spawn_generation {
6060                sleep(Duration::from_millis(5)).await;
6061            }
6062        })
6063        .await
6064        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6065        module.drain().await.unwrap();
6066        server.abort();
6067    }
6068
6069    #[cfg(unix)]
6070    #[tokio::test]
6071    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6072        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6073        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6074        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6075        drop(unused);
6076        let configured = http_fixture(&dir, &url, 2);
6077        let module =
6078            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6079                .supervise_configured_with_health(
6080                    configured.module_spec(),
6081                    true,
6082                    configured.health,
6083                    configured.drain_timeout_ms,
6084                    configured.restart,
6085                )
6086                .unwrap();
6087        let before = module.status().unwrap().spawn_generation;
6088        timeout(Duration::from_secs(5), async {
6089            loop {
6090                let status = module.status().unwrap();
6091                if status.spawn_generation > before {
6092                    assert!(status.lifetime_restarts > 0);
6093                    break;
6094                }
6095                sleep(Duration::from_millis(5)).await;
6096            }
6097        })
6098        .await
6099        .expect("sustained HTTP refusal must trigger the health restart policy");
6100        module.drain().await.unwrap();
6101    }
6102
6103    #[cfg(unix)]
6104    #[tokio::test]
6105    async fn http_health_timeout_honours_deadline() {
6106        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6107        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6108        let server = tokio::spawn(async move {
6109            let _held = listener.accept().await.unwrap();
6110            std::future::pending::<()>().await;
6111        });
6112        let error = timeout(
6113            Duration::from_secs(1),
6114            probe_http_health(&url, Duration::from_millis(10)),
6115        )
6116        .await
6117        .expect("the probe must enforce its own deadline")
6118        .unwrap_err();
6119        server.abort();
6120        assert!(error.to_string().contains("timed out"));
6121    }
6122
6123    #[cfg(unix)]
6124    #[tokio::test]
6125    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6126        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6127        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6128        let url = format!(
6129            "http://localhost:{}/healthz",
6130            listener.local_addr().unwrap().port()
6131        );
6132        let server = tokio::spawn(async move {
6133            let (mut stream, _) = listener.accept().await.unwrap();
6134            let mut request = [0u8; 2048];
6135            assert!(stream.read(&mut request).await.unwrap() > 0);
6136            stream
6137                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6138                .await
6139                .unwrap();
6140        });
6141        // The deadline only bounds a hang. A probe that never tried the IPv6
6142        // address would be refused on 127.0.0.1 and fail at once, so a longer
6143        // deadline does not weaken the assertion; one second timed out under a
6144        // loaded parallel test run.
6145        assert_eq!(
6146            probe_http_health(&url, Duration::from_secs(10))
6147                .await
6148                .unwrap()
6149                .status,
6150            HealthStatus::Ok
6151        );
6152        server.await.unwrap();
6153    }
6154
6155    #[cfg(unix)]
6156    #[tokio::test]
6157    async fn http_health_timeout_keeps_partial_status_and_body() {
6158        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6159        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6160        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6161        let server = tokio::spawn(async move {
6162            let (mut stream, _) = listener.accept().await.unwrap();
6163            let mut request = [0u8; 2048];
6164            assert!(stream.read(&mut request).await.unwrap() > 0);
6165            stream
6166                .write_all(
6167                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6168                )
6169                .await
6170                .unwrap();
6171            std::future::pending::<()>().await;
6172        });
6173        let error = probe_http_health(&url, Duration::from_secs(1))
6174            .await
6175            .unwrap_err()
6176            .to_string();
6177        server.abort();
6178        assert!(
6179            error.contains("timed out")
6180                && error.contains("503 Unavailable")
6181                && error.contains("partial diagnostic"),
6182            "{error}"
6183        );
6184    }
6185
6186    #[cfg(unix)]
6187    #[tokio::test]
6188    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6189        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6190        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6191        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6192        let server = tokio::spawn(async move {
6193            let (mut stream, _) = listener.accept().await.unwrap();
6194            let mut request = [0u8; 2048];
6195            assert!(stream.read(&mut request).await.unwrap() > 0);
6196            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6197            let response = format!(
6198                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6199                body.len()
6200            );
6201            stream.write_all(response.as_bytes()).await.unwrap();
6202        });
6203        let error = probe_http_health(&url, Duration::from_secs(1))
6204            .await
6205            .unwrap_err()
6206            .to_string();
6207        server.await.unwrap();
6208        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6209        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6210        assert!(!error.contains("not-in-diagnostic"));
6211    }
6212
6213    #[cfg(unix)]
6214    #[tokio::test]
6215    async fn http_health_real_nats_server_monitoring() {
6216        if std::process::Command::new("nats-server")
6217            .arg("--version")
6218            .env("XDG_DATA_HOME", std::env::temp_dir())
6219            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6220            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6221            .output()
6222            .is_err()
6223        {
6224            eprintln!(
6225                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6226            );
6227            return;
6228        }
6229        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6230        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6231        let port = monitor.local_addr().unwrap().port();
6232        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6233        let client_port = client.local_addr().unwrap().port();
6234        let config = dir.join("server.conf");
6235        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6236        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6237        configured.program = PathBuf::from("nats-server");
6238        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6239        drop(monitor);
6240        drop(client);
6241        let module =
6242            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6243                .supervise_configured_with_health(
6244                    configured.module_spec(),
6245                    true,
6246                    configured.health,
6247                    configured.drain_timeout_ms,
6248                    configured.restart,
6249                )
6250                .unwrap();
6251        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6252        module.drain().await.unwrap();
6253    }
6254
6255    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6256    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6257        let supervisor =
6258            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6259        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6260        let initial = ModuleSpec {
6261            module_id: "rescan-preserves-spawn-facts".to_string(),
6262            program: PathBuf::from("/spawned/module"),
6263            args: Vec::new(),
6264            env: Vec::new(),
6265            reserved: false,
6266            reserved_prefixes: Vec::new(),
6267            protocol: ModuleProtocol::Subc,
6268            overlap: Default::default(),
6269        };
6270        let module = supervisor.supervised_module(
6271            initial.clone(),
6272            supervisor.runtime_config(),
6273            snapshot,
6274            None,
6275        );
6276        let before = module.status().unwrap();
6277        let mut replacement = initial;
6278        replacement.program = PathBuf::from("/rescanned/replacement-module");
6279
6280        module
6281            .update_configuration(replacement, HealthConfig::default(), None)
6282            .await
6283            .unwrap();
6284
6285        let after = module.status().unwrap();
6286        assert_eq!(after.pid, before.pid);
6287        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6288        assert_eq!(after.spawned_from, before.spawned_from);
6289        drop(module);
6290    }
6291}
6292
6293fn unix_ms_now() -> u64 {
6294    SystemTime::now()
6295        .duration_since(UNIX_EPOCH)
6296        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6297        .unwrap_or(0)
6298}
6299
6300async fn supervise_loop(
6301    mut spec: ModuleSpec,
6302    mut runtime: SupervisorRuntimeConfig,
6303    registry: Arc<Registry>,
6304    process_liveness: Arc<SupervisorProcessLiveness>,
6305    snapshot: SharedSnapshot,
6306    mut child: Option<SupervisedChild>,
6307    mut commands: mpsc::Receiver<SupervisorCommand>,
6308) {
6309    let mut health_probe = HealthProbeRuntime::default();
6310    // Registry writes (including embedded callers) notify this module only.
6311    // Subscribe before the first refresh; watch retains changes that arrive
6312    // while commands or probes are running. No polling fallback is needed.
6313    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6314        Ok(changes) => Some(changes),
6315        Err(err) => {
6316            // A poisoned registry cannot accept further writes, so it cannot
6317            // recover via a timer. Keep exit and command handling alive.
6318            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6319            None
6320        }
6321    };
6322    // All restart backoffs run here, including health and operator requests.
6323    // While one is pending the loop serves commands, so disable or drain can
6324    // cancel the replacement without spawning a process just to stop it.
6325    let mut pending_respawn: Option<PendingRespawn> = None;
6326    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
6327    // before anything else so a stop that interrupted a swap runs at once.
6328    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6329    let mut previous_instance_recheck = tokio::time::interval_at(
6330        Instant::now() + Duration::from_secs(5),
6331        Duration::from_secs(5),
6332    );
6333    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6334    loop {
6335        #[cfg(test)]
6336        {
6337            lock_snapshot(&snapshot).unwrap().actor_select = None;
6338        }
6339        #[cfg(target_os = "macos")]
6340        if let Some(active) = child.as_mut() {
6341            active.confirm_privacy_exec().await;
6342        }
6343        if let Some(scheduled) = runtime
6344            .scheduled_respawn
6345            .lock()
6346            .unwrap_or_else(|p| p.into_inner())
6347            .take()
6348        {
6349            pending_respawn = Some(scheduled);
6350        }
6351        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6352            pending_respawn = None;
6353            cancel_deferred_reload(
6354                &runtime,
6355                &spec.module_id,
6356                "respawn cancelled by a supervisor command",
6357            );
6358        }
6359        if child.is_none()
6360            && pending_respawn.is_none()
6361            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6362        {
6363            cancel_deferred_reload(
6364                &runtime,
6365                &spec.module_id,
6366                "respawn cancelled before a replacement was spawned",
6367            );
6368            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6369                state.respawn_pending = false;
6370                state.coalesced_restart_pending = false;
6371                if matches!(
6372                    state.state,
6373                    ModuleState::Restarting
6374                        | ModuleState::Starting
6375                        | ModuleState::Draining
6376                        | ModuleState::Unresponsive
6377                ) {
6378                    error!(module_id = %spec.module_id, state = ?state.state,
6379                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6380                    state.state = ModuleState::Failed;
6381                    clear_current_process_facts(state);
6382                }
6383            });
6384        }
6385        if let Some(command) = requeued.pop_front() {
6386            if !handle_supervisor_command(
6387                command,
6388                &mut spec,
6389                &mut runtime,
6390                &registry,
6391                &process_liveness,
6392                &snapshot,
6393                &mut child,
6394                &mut commands,
6395                &mut requeued,
6396            )
6397            .await
6398            {
6399                return;
6400            }
6401            if child.is_some() || !respawn_still_pending(&snapshot) {
6402                pending_respawn = None;
6403                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6404                    state.respawn_pending = false
6405                });
6406            }
6407            continue;
6408        }
6409        if child.is_some() {
6410            #[cfg(test)]
6411            {
6412                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6413            }
6414            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
6415            let wake_after = health_probe.wake_after();
6416            let probe_sleep = async move {
6417                match wake_after {
6418                    Some(delay) => sleep(delay).await,
6419                    None => std::future::pending().await,
6420                }
6421            };
6422            tokio::pin!(probe_sleep);
6423            // A `protocol: "none"` child (a plain process that never registers
6424            // over the subc wire, such as nats-server) with no HTTP health
6425            // check has nothing to probe, so it waits only for its exit or a
6426            // supervisor command. A rescan that adds an HTTP check is a command.
6427            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6428            #[cfg(test)]
6429            {
6430                let registration_pending = wire_child
6431                    && registration_changes
6432                        .as_ref()
6433                        .is_some_and(|changes| changes.has_changed().unwrap());
6434                let mut state = lock_snapshot(&snapshot).unwrap();
6435                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6436                    generation: state.spawn_generation,
6437                    turn: state.actor_turns,
6438                    registered_connection: health_probe.registered_connection,
6439                    next_probe_at: health_probe.next_probe_at,
6440                    wake_after,
6441                });
6442            }
6443            let active_child = child.as_mut().expect("child checked above");
6444            tokio::select! {
6445                wait_result = active_child.wait() => {
6446                    #[cfg(test)]
6447                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6448                    // Every arm below that gives up on the CHILD must keep the
6449                    // supervision task itself alive (child = None, loop
6450                    // continues into command-serving mode). Returning here
6451                    // closes the command channel, which makes the module
6452                    // permanently unrestartable in-band: a clean child exit
6453                    // of an enabled module once wedged the fleet this way
6454                    // ('supervisor command channel is closed') and required a
6455                    // full daemon restart to recover.
6456                    let exit_report = match wait_result {
6457                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6458                        Err(err) => {
6459                            active_child.drain_stderr(&spec.module_id).await;
6460                            fail_after_wait_error(
6461                                &spec.module_id,
6462                                &snapshot,
6463                                &runtime.terminal_ring,
6464                                &runtime.spawn_events,
6465                                format!("failed to wait for supervised module: {err}"),
6466                            );
6467                            untrack_if_registration_released(
6468                                &process_liveness,
6469                                &registry,
6470                                &spec.module_id,
6471                                &snapshot,
6472                            );
6473                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6474                            child = None;
6475                            continue;
6476                        }
6477                    };
6478                    active_child.drain_stderr(&spec.module_id).await;
6479
6480                    let next = on_child_exit(
6481                        &spec,
6482                        runtime.restart_policy,
6483                        &registry,
6484                        &snapshot,
6485                        &runtime.terminal_ring,
6486                        &runtime.spawn_events,
6487                        &runtime.child_roster,
6488                        exit_report,
6489                    ).await;
6490                    runtime.start_requester = None;
6491                    runtime.pending_activity_restart = false;
6492                    // The exit is recorded, so a daemon shutdown may stop
6493                    // waiting for this child (see `SupervisedChild::wait`).
6494                    active_child.release_roster();
6495                    match next {
6496                        NextAction::Stop { registration_released } => {
6497                            if registration_released {
6498                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6499                            }
6500                            child = None;
6501                        }
6502                        NextAction::Restart { schedule } => {
6503                            let delay = schedule.map_or(
6504                                runtime.restart_policy.delay_for_restart(0),
6505                                |schedule| schedule.delay,
6506                            );
6507                            if let Some(schedule) = schedule {
6508                                log_crash_respawn(&spec.module_id, schedule);
6509                            }
6510                            // The exited child is fully recorded at this point,
6511                            // so release it and count the backoff down in the
6512                            // command-serving branch below rather than sleeping
6513                            // here: commands cannot be received from inside this
6514                            // select arm, and an operator disable or drain that
6515                            // arrives during the backoff must cancel the pending
6516                            // respawn instead of waiting for it to spawn first.
6517                            child = None;
6518                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6519                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6520                        }
6521                    }
6522                }
6523                command = commands.recv() => {
6524                    #[cfg(test)]
6525                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6526                    let Some(command) = command else {
6527                        return;
6528                    };
6529                    if !handle_supervisor_command(
6530                        command,
6531                        &mut spec,
6532                        &mut runtime,
6533                        &registry,
6534                        &process_liveness,
6535                        &snapshot,
6536                        &mut child,
6537                        &mut commands,
6538                        &mut requeued,
6539                    ).await {
6540                        return;
6541                    }
6542                }
6543                _ = async {
6544                    match registration_changes.as_mut() {
6545                        Some(changes) => { let _ = changes.changed().await; }
6546                        None => std::future::pending().await,
6547                    }
6548                }, if wire_child => {
6549                    #[cfg(test)]
6550                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6551                }
6552                _ = &mut probe_sleep => {
6553                    #[cfg(test)]
6554                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6555                    if health_probe.due() {
6556                        run_health_probe_cycle(
6557                            &spec,
6558                            &runtime,
6559                            &registry,
6560                            &process_liveness,
6561                            &snapshot,
6562                            &mut child,
6563                        ).await;
6564                        if child.is_none() {
6565                            runtime.start_requester = None;
6566                            runtime.pending_activity_restart = false;
6567                        }
6568                        if child.is_some() {
6569                            health_probe.schedule_next(&spec, runtime.health.cadence);
6570                        }
6571                    }
6572                }
6573            }
6574        } else if let Some(pending) = pending_respawn {
6575            tokio::select! {
6576                _ = sleep_until(pending.deadline) => {
6577                    pending_respawn = None;
6578                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6579                    // A command handled below while the backoff elapsed may
6580                    // have stopped the module; never respawn past an operator's
6581                    // disable or drain.
6582                    if !respawn_still_pending(&snapshot) {
6583                        continue;
6584                    }
6585                    // The daemon began shutting down during the backoff: the
6586                    // spawn would be refused anyway, and refusing it here
6587                    // leaves the module stopped instead of reporting a
6588                    // failed restart.
6589                    if runtime.child_roster.is_closed() {
6590                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6591                            state.state = ModuleState::Stopped;
6592                        });
6593                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6594                        continue;
6595                    }
6596                    if let Err(err) = release_dead_registration(
6597                        &registry,
6598                        runtime.forwarding.as_deref(),
6599                        &snapshot,
6600                        &spec.module_id,
6601                    ).await {
6602                        fail_snapshot(&snapshot, Some(&spec.module_id), None);
6603                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6604                        continue;
6605                    }
6606
6607                    if matches!(pending.kind, RespawnKind::Reload) {
6608                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6609                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
6610                        if result.is_ok() && runtime.pending_activity_restart {
6611                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6612                        }
6613                        runtime.pending_activity_restart = false;
6614                        if let Some(reply) = reply { let _ = reply.send(result); }
6615                        continue;
6616                    }
6617                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6618                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6619                        Ok(next_child) => {
6620                            child = Some(next_child);
6621                            if runtime.pending_activity_restart {
6622                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6623                                runtime.pending_activity_restart = false;
6624                            }
6625                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6626                        }
6627                        Err(err) => {
6628                            fail_snapshot(&snapshot, Some(&spec.module_id), None);
6629                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6630                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6631                        }
6632                    }
6633                }
6634                command = commands.recv() => {
6635                    let Some(command) = command else {
6636                        return;
6637                    };
6638                    if !handle_supervisor_command(
6639                        command,
6640                        &mut spec,
6641                        &mut runtime,
6642                        &registry,
6643                        &process_liveness,
6644                        &snapshot,
6645                        &mut child,
6646                        &mut commands,
6647                        &mut requeued,
6648                    ).await {
6649                        return;
6650                    }
6651                    // Reconcile the pending respawn with what the command did:
6652                    // a start may already have spawned a fresh child,
6653                    // while a disable or drain moved the snapshot out of the
6654                    // state the respawn was counting down from.
6655                    if child.is_some() || !respawn_still_pending(&snapshot) {
6656                        pending_respawn = None;
6657                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6658                    }
6659                }
6660            }
6661        } else {
6662            let command = if lock_snapshot(&snapshot)
6663                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
6664            {
6665                tokio::select! {
6666                    command = commands.recv() => command,
6667                    _ = previous_instance_recheck.tick() => {
6668                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
6669                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
6670                            if enabled && !runtime.child_roster.is_closed() {
6671                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6672                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6673                                    Ok(next_child) => child = Some(next_child),
6674                                    Err(error) => {
6675                                        fail_snapshot(&snapshot, Some(&spec.module_id), None);
6676                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
6677                                    }
6678                                }
6679                            } else {
6680                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6681                                    state.waiting_for_previous_pid = None;
6682                                    state.waiting_for_previous_detail = None;
6683                                });
6684                            }
6685                        }
6686                        continue;
6687                    }
6688                }
6689            } else {
6690                commands.recv().await
6691            };
6692            let Some(command) = command else {
6693                return;
6694            };
6695            if !handle_supervisor_command(
6696                command,
6697                &mut spec,
6698                &mut runtime,
6699                &registry,
6700                &process_liveness,
6701                &snapshot,
6702                &mut child,
6703                &mut commands,
6704                &mut requeued,
6705            )
6706            .await
6707            {
6708                return;
6709            }
6710        }
6711    }
6712}
6713
6714fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
6715    info!(
6716        module_id,
6717        restart_in_window = schedule.restart_in_window,
6718        delay_ms = schedule.delay.as_millis() as u64,
6719        "respawning after crash"
6720    );
6721}
6722
6723/// Whether the respawn a backoff was counting down to is still wanted. A
6724/// disable or drain handled while the backoff elapsed moves the snapshot out
6725/// of `Restarting`, and the operator's stop must win over the pending respawn,
6726/// so every sleep-then-spawn path re-validates against the live snapshot
6727/// instead of assuming the state it left behind still holds.
6728fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
6729    matches!(
6730        lock_snapshot(snapshot),
6731        Ok(state) if state.enabled && state.state == ModuleState::Restarting
6732    )
6733}
6734
6735enum NextAction {
6736    Stop {
6737        registration_released: bool,
6738    },
6739    Restart {
6740        schedule: Option<CrashRestartSchedule>,
6741    },
6742}
6743
6744#[allow(clippy::too_many_arguments)]
6745async fn handle_supervisor_command(
6746    command: SupervisorCommand,
6747    spec: &mut ModuleSpec,
6748    runtime: &mut SupervisorRuntimeConfig,
6749    registry: &Arc<Registry>,
6750    process_liveness: &SupervisorProcessLiveness,
6751    snapshot: &SharedSnapshot,
6752    child: &mut Option<SupervisedChild>,
6753    commands: &mut mpsc::Receiver<SupervisorCommand>,
6754    requeued: &mut VecDeque<SupervisorCommand>,
6755) -> bool {
6756    let command =
6757        if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
6758            match command {
6759                SupervisorCommand::Restart { reply, .. }
6760                | SupervisorCommand::Reload { reply, .. }
6761                | SupervisorCommand::Swap { reply, .. } => {
6762                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6763                    return true;
6764                }
6765                SupervisorCommand::SetEnabled {
6766                    enabled: true,
6767                    reply,
6768                    ..
6769                } => {
6770                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6771                    return true;
6772                }
6773                other => other,
6774            }
6775        } else {
6776            command
6777        };
6778    match command {
6779        SupervisorCommand::Drain { requester, reply } => {
6780            // A plain stop runs no forwarding drain, so nothing reaches the
6781            // module over its connection before the wait: ask by signal.
6782            let result = drain_optional_child(
6783                &spec.module_id,
6784                spec.protocol,
6785                StopNotice::NotSent,
6786                registry,
6787                runtime.forwarding.as_deref(),
6788                snapshot,
6789                &runtime.terminal_ring,
6790                &runtime.spawn_events,
6791                child,
6792                runtime.drain_timeout,
6793                ModuleState::Stopped,
6794                None,
6795            )
6796            .await;
6797            let registration_released = result.is_ok();
6798            if registration_released {
6799                runtime.record_activity_transition(
6800                    "subc.module_stopped",
6801                    &spec.module_id,
6802                    requester,
6803                );
6804            }
6805            let _ = reply.send(result);
6806            if registration_released {
6807                process_liveness.untrack_if_current(&spec.module_id, snapshot);
6808            }
6809            false
6810        }
6811        SupervisorCommand::Retire { requester, reply } => {
6812            let result = async {
6813                let stop_notice = begin_forwarding_drain_if_configured(
6814                    spec,
6815                    runtime,
6816                    registry,
6817                    snapshot,
6818                    None,
6819                    RouteCloseReason::Disable,
6820                )
6821                .await?;
6822                drain_optional_child(
6823                    &spec.module_id,
6824                    spec.protocol,
6825                    stop_notice,
6826                    registry,
6827                    runtime.forwarding.as_deref(),
6828                    snapshot,
6829                    &runtime.terminal_ring,
6830                    &runtime.spawn_events,
6831                    child,
6832                    runtime.drain_timeout,
6833                    ModuleState::Stopped,
6834                    None,
6835                )
6836                .await
6837            }
6838            .await;
6839            let registration_released = result.is_ok();
6840            if registration_released {
6841                runtime.record_activity_transition(
6842                    "subc.module_stopped",
6843                    &spec.module_id,
6844                    requester,
6845                );
6846            }
6847            let _ = reply.send(result);
6848            if registration_released {
6849                process_liveness.untrack_if_current(&spec.module_id, snapshot);
6850            }
6851            false
6852        }
6853        SupervisorCommand::Restart {
6854            requester,
6855            drain_timeout_ms,
6856            received_at_generation,
6857            queued_at,
6858            reply,
6859        } => {
6860            // Without this line a restart that waited in the queue (behind a
6861            // health probe cycle or another command) was invisible: the log
6862            // showed only the drain timing out, minutes after the operator's call.
6863            info!(
6864                module_id = %spec.module_id,
6865                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
6866                "restart command dequeued"
6867            );
6868            // ACK AT INITIATION, not completion. The blocking form deadlocked any
6869            // caller whose own request lane rides the module being restarted: the
6870            // caller's in-flight request keeps the drain from quiescing, the drain
6871            // keeps the restart from completing, and the completion keeps the reply
6872            // from releasing the caller — so the drain always timed out and cut the
6873            // initiator with a GOODBYE, even on a healthy module. Replying once the
6874            // restart is validated lets a self-lane caller settle, which is exactly
6875            // what makes the drain succeed. Completion is observable via
6876            // supervisor.list / module status; a post-ack failure lands the module
6877            // in a visible terminal state below rather than in a reply nobody can
6878            // receive.
6879            let validation = match lock_snapshot(snapshot) {
6880                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
6881                    module_id: spec.module_id.clone(),
6882                }),
6883                Ok(_) => Ok(()),
6884                Err(err) => Err(err),
6885            };
6886            let initiated = validation.is_ok();
6887            let _ = reply.send(validation);
6888            // A restart asks for a fresh process. Commands run one at a time,
6889            // so a restart queued behind another restart (two operator calls
6890            // in quick succession) is dequeued the moment the first one has
6891            // spawned its replacement -- before that process has sent HELLO.
6892            // Running it would drain and kill the process the first restart
6893            // just produced, which is the opposite of what both callers asked
6894            // for. If a process spawned after this request was received is
6895            // still supervised, the request is already satisfied. Not when the
6896            // configuration changed since that spawn: then the newer process
6897            // predates the spec this restart may exist to apply.
6898            let satisfied_by_generation = if initiated && child.is_some() {
6899                lock_snapshot(snapshot).ok().and_then(|state| {
6900                    (state.spawn_generation > received_at_generation
6901                        && !state.configuration_updated_since_spawn)
6902                        .then_some(state.spawn_generation)
6903                })
6904            } else {
6905                None
6906            };
6907            let satisfied_by_pending = initiated
6908                && child.is_none()
6909                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
6910                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
6911                    if pending {
6912                        state.coalesced_restart_pending = true;
6913                    }
6914                    pending
6915                });
6916            if satisfied_by_pending {
6917                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
6918            } else if let Some(generation) = satisfied_by_generation {
6919                info!(
6920                    module_id = %spec.module_id,
6921                    received_at_generation,
6922                    "restart already satisfied by generation {generation}; not restarting again"
6923                );
6924            } else if initiated {
6925                // Precedence: this restart's operator override, else the module's
6926                // configured budget (already resolved into the runtime).
6927                let drain_timeout = drain_timeout_ms
6928                    .map(Duration::from_millis)
6929                    .unwrap_or(runtime.drain_timeout);
6930                if let Err(err) = restart_child(
6931                    spec,
6932                    runtime,
6933                    registry,
6934                    process_liveness,
6935                    snapshot,
6936                    child,
6937                    drain_timeout,
6938                )
6939                .await
6940                {
6941                    warn!(
6942                        module_id = %spec.module_id,
6943                        error = %err,
6944                        "operator restart failed after initiation ack; module state carries the outcome"
6945                    );
6946                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6947                        state.state = ModuleState::Failed;
6948                        clear_current_process_facts(state);
6949                    });
6950                } else {
6951                    runtime.start_requester = requester;
6952                    runtime.pending_activity_restart = true;
6953                }
6954            }
6955            true
6956        }
6957        SupervisorCommand::Reload { requester, reply } => {
6958            let result =
6959                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
6960            if result.is_ok() {
6961                runtime.start_requester = requester;
6962                runtime.pending_activity_restart = true;
6963            }
6964            if result.is_ok()
6965                && runtime
6966                    .scheduled_respawn
6967                    .lock()
6968                    .unwrap_or_else(|p| p.into_inner())
6969                    .as_ref()
6970                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
6971            {
6972                *runtime
6973                    .deferred_reload_reply
6974                    .lock()
6975                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
6976            } else {
6977                let _ = reply.send(result);
6978            }
6979            true
6980        }
6981        SupervisorCommand::SetEnabled {
6982            enabled,
6983            requester,
6984            reply,
6985        } => {
6986            let previous_requester = runtime.start_requester.clone();
6987            if enabled {
6988                runtime.start_requester = requester.clone();
6989            }
6990            let result = set_child_enabled(
6991                spec,
6992                runtime,
6993                registry,
6994                process_liveness,
6995                snapshot,
6996                child,
6997                enabled,
6998            )
6999            .await;
7000            if matches!(result, Ok(true)) {
7001                runtime.pending_activity_restart = false;
7002                if enabled {
7003                    runtime.start_requester = requester;
7004                } else {
7005                    runtime.start_requester = None;
7006                    runtime.record_activity_transition(
7007                        "subc.module_disabled",
7008                        &spec.module_id,
7009                        requester,
7010                    );
7011                }
7012            } else {
7013                runtime.start_requester = previous_requester;
7014            }
7015            let _ = reply.send(result);
7016            true
7017        }
7018        SupervisorCommand::UpdateConfiguration {
7019            spec: next_spec,
7020            health,
7021            drain_timeout_ms,
7022            reply,
7023        } => {
7024            if let Some(handle) = &runtime.supervisor_handle {
7025                handle.apply_identity_configuration(&next_spec);
7026            }
7027            *spec = next_spec;
7028            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7029                state.configuration_updated_since_spawn = true;
7030            });
7031            let health_changed = runtime.health != health;
7032            runtime.health = health;
7033            // Reset the cadence and old endpoint's failure streak on a live
7034            // health-policy change rather than waiting for its old deadline.
7035            if health_changed {
7036                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7037                    state.health = ModuleHealthStatus::default();
7038                });
7039            }
7040            runtime.drain_timeout = drain_timeout_ms
7041                .map(Duration::from_millis)
7042                .unwrap_or(runtime.default_drain_timeout);
7043            *runtime
7044                .effective_drain_timeout
7045                .lock()
7046                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7047            let _ = reply.send(());
7048            true
7049        }
7050        SupervisorCommand::Swap {
7051            requester,
7052            ready_timeout,
7053            reply,
7054        } => {
7055            let mut swap_runtime = runtime.clone();
7056            swap_runtime.start_requester = requester;
7057            let before = lock_snapshot(snapshot)
7058                .ok()
7059                .map(|state| state.spawn_generation);
7060            let end = swap::run_swap(
7061                spec,
7062                &swap_runtime,
7063                registry,
7064                process_liveness,
7065                snapshot,
7066                child,
7067                commands,
7068                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7069                reply,
7070            )
7071            .await;
7072            if before
7073                != lock_snapshot(snapshot)
7074                    .ok()
7075                    .map(|state| state.spawn_generation)
7076            {
7077                runtime.start_requester = swap_runtime.start_requester;
7078            }
7079            requeued.extend(end.requeue);
7080            true
7081        }
7082    }
7083}
7084
7085async fn restart_child(
7086    spec: &ModuleSpec,
7087    runtime: &SupervisorRuntimeConfig,
7088    registry: &Registry,
7089    process_liveness: &SupervisorProcessLiveness,
7090    snapshot: &SharedSnapshot,
7091    child: &mut Option<SupervisedChild>,
7092    drain_timeout: Duration,
7093) -> Result<(), SuperviseError> {
7094    // Restart cycles a running module; it must not silently start a disabled one.
7095    if !lock_snapshot(snapshot)?.enabled {
7096        return Err(SuperviseError::Disabled {
7097            module_id: spec.module_id.clone(),
7098        });
7099    }
7100    let stop_notice = begin_forwarding_drain_with_timeout(
7101        spec,
7102        runtime,
7103        registry,
7104        snapshot,
7105        None,
7106        RouteCloseReason::Restart,
7107        drain_timeout,
7108    )
7109    .await?;
7110
7111    if child.is_some() {
7112        drain_optional_child(
7113            &spec.module_id,
7114            spec.protocol,
7115            stop_notice,
7116            registry,
7117            runtime.forwarding.as_deref(),
7118            snapshot,
7119            &runtime.terminal_ring,
7120            &runtime.spawn_events,
7121            child,
7122            drain_timeout,
7123            ModuleState::Restarting,
7124            Some(true),
7125        )
7126        .await?;
7127    } else {
7128        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7129            state.enabled = true;
7130            state.state = ModuleState::Restarting;
7131            clear_current_process_facts(state);
7132        })?;
7133        release_dead_registration(
7134            registry,
7135            runtime.forwarding.as_deref(),
7136            snapshot,
7137            &spec.module_id,
7138        )
7139        .await?;
7140    }
7141
7142    reset_restart_count(snapshot, &spec.module_id)?;
7143    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7144    schedule_respawn(
7145        runtime,
7146        snapshot,
7147        &spec.module_id,
7148        runtime.restart_policy.backoff,
7149        RespawnKind::Spawn,
7150    )
7151}
7152
7153async fn reload_child(
7154    spec: &ModuleSpec,
7155    runtime: &SupervisorRuntimeConfig,
7156    registry: &Registry,
7157    process_liveness: &SupervisorProcessLiveness,
7158    snapshot: &SharedSnapshot,
7159    child: &mut Option<SupervisedChild>,
7160) -> Result<(), SuperviseError> {
7161    // Reload cycles a running module; it must not silently start a disabled one.
7162    if !lock_snapshot(snapshot)?.enabled {
7163        return Err(SuperviseError::Disabled {
7164            module_id: spec.module_id.clone(),
7165        });
7166    }
7167    let stop_notice = begin_forwarding_drain(
7168        spec,
7169        runtime,
7170        registry,
7171        snapshot,
7172        Some(true),
7173        RouteCloseReason::Reload,
7174    )
7175    .await?;
7176
7177    if child.is_some() {
7178        drain_optional_child(
7179            &spec.module_id,
7180            spec.protocol,
7181            stop_notice,
7182            registry,
7183            runtime.forwarding.as_deref(),
7184            snapshot,
7185            &runtime.terminal_ring,
7186            &runtime.spawn_events,
7187            child,
7188            runtime.drain_timeout,
7189            ModuleState::Restarting,
7190            Some(true),
7191        )
7192        .await?;
7193    } else {
7194        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7195            state.enabled = true;
7196            state.state = ModuleState::Restarting;
7197            clear_current_process_facts(state);
7198        })?;
7199        release_dead_registration(
7200            registry,
7201            runtime.forwarding.as_deref(),
7202            snapshot,
7203            &spec.module_id,
7204        )
7205        .await?;
7206    }
7207
7208    reset_restart_count(snapshot, &spec.module_id)?;
7209    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7210    schedule_respawn(
7211        runtime,
7212        snapshot,
7213        &spec.module_id,
7214        runtime.restart_policy.backoff,
7215        RespawnKind::Reload,
7216    )
7217}
7218
7219async fn finish_reload_child(
7220    spec: &ModuleSpec,
7221    runtime: &SupervisorRuntimeConfig,
7222    registry: &Registry,
7223    process_liveness: &SupervisorProcessLiveness,
7224    snapshot: &SharedSnapshot,
7225    child: &mut Option<SupervisedChild>,
7226) -> Result<(), SuperviseError> {
7227    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7228    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7229        Ok(next_child) => next_child,
7230        Err(err) => {
7231            return handle_reload_spawn_failure(
7232                spec,
7233                runtime,
7234                process_liveness,
7235                snapshot,
7236                child,
7237                format!("new child failed to spawn: {err}"),
7238            )
7239            .await;
7240        }
7241    };
7242    *child = Some(next_child);
7243
7244    let wait_outcome = {
7245        let active_child = child.as_mut().expect("new reload child was just stored");
7246        wait_for_registration_after_reload(
7247            registry,
7248            &spec.module_id,
7249            snapshot,
7250            active_child,
7251            REGISTRY_RELEASE_TIMEOUT,
7252        )
7253        .await?
7254    };
7255
7256    match wait_outcome {
7257        RegistrationWaitOutcome::Registered => {
7258            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7259            Ok(())
7260        }
7261        RegistrationWaitOutcome::Exited(exit_report) => {
7262            if let Some(active_child) = child.as_mut() {
7263                active_child.drain_stderr(&spec.module_id).await;
7264            }
7265            // Keep the reaped child's roster guard until its terminal is written.
7266            // Shutdown waits on that guard, not on the child Option used for respawn.
7267            let mut exited_child = child.take().expect("exited reload child is still stored");
7268            #[cfg(test)]
7269            if let Some(gate) = &runtime.test_reload_exit_record_gate {
7270                gate.reached.notify_one();
7271                gate.resume.notified().await;
7272            }
7273            let result = handle_reload_child_registration_failure(
7274                spec,
7275                runtime,
7276                registry,
7277                process_liveness,
7278                snapshot,
7279                child,
7280                ReloadRegistrationFailure {
7281                    exit_report: registration_failure_exit_report(exit_report),
7282                    reason: exited_child
7283                        .spawn_failure
7284                        .clone()
7285                        .unwrap_or_else(|| "new child exited before registering".to_string()),
7286                },
7287            )
7288            .await;
7289            exited_child.release_roster();
7290            result
7291        }
7292        RegistrationWaitOutcome::TimedOut => {
7293            let mut timed_out_child = child
7294                .take()
7295                .expect("timed-out reload child is still running");
7296            timed_out_child
7297                .start_kill()
7298                .map_err(|source| SuperviseError::Kill {
7299                    module_id: spec.module_id.clone(),
7300                    source,
7301                })?;
7302            let status = timed_out_child
7303                .wait()
7304                .await
7305                .map_err(|source| SuperviseError::Wait {
7306                    module_id: spec.module_id.clone(),
7307                    source,
7308                })?;
7309            timed_out_child.drain_stderr(&spec.module_id).await;
7310            handle_reload_child_registration_failure(
7311                spec,
7312                runtime,
7313                registry,
7314                process_liveness,
7315                snapshot,
7316                child,
7317                ReloadRegistrationFailure {
7318                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7319                        snapshot,
7320                        &timed_out_child,
7321                        &status,
7322                    )),
7323                    reason: format!(
7324                        "new child did not register within {:?}",
7325                        REGISTRY_RELEASE_TIMEOUT
7326                    ),
7327                },
7328            )
7329            .await
7330        }
7331    }
7332}
7333
7334async fn set_child_enabled(
7335    spec: &ModuleSpec,
7336    runtime: &SupervisorRuntimeConfig,
7337    registry: &Registry,
7338    process_liveness: &SupervisorProcessLiveness,
7339    snapshot: &SharedSnapshot,
7340    child: &mut Option<SupervisedChild>,
7341    enabled: bool,
7342) -> Result<bool, SuperviseError> {
7343    let (current_enabled, current_state, respawn_pending) = {
7344        let state = lock_snapshot(snapshot)?;
7345        (state.enabled, state.state, state.respawn_pending)
7346    };
7347    // `start` (enable on an already-enabled module) heals TERMINAL states instead
7348    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
7349    // clean (Stopped) has no live process and no other in-band recovery — the
7350    // operator's start is the explicit recovery act and resets the budget. Without
7351    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
7352    // which the 2026-07-14 aft outage proved is a trap when the failed module is
7353    // the one providing every agent's shell.
7354    let revive_terminal = enabled
7355        && current_enabled
7356        && child.is_none()
7357        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7358            || (current_state == ModuleState::Restarting && !respawn_pending));
7359    if current_enabled == enabled && !revive_terminal {
7360        return Ok(false);
7361    }
7362
7363    if enabled {
7364        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7365            state.enabled = true;
7366            state.state = ModuleState::Starting;
7367            clear_current_process_facts(state);
7368        })?;
7369        #[cfg(test)]
7370        if runtime.test_seed_stale_facts_before_enable_spawn {
7371            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7372                state.process_alive = true;
7373                state.pid = Some(41);
7374                state.spawned_at_ms = Some(42);
7375                state.spawned_from = Some(PathBuf::from("/spawned/module"));
7376                state.spawned_file_identity = Some(SpawnedFileIdentity {
7377                    device: 43,
7378                    inode: 44,
7379                    #[cfg(windows)]
7380                    windows_image: None,
7381                });
7382            })?;
7383        }
7384        release_dead_registration(
7385            registry,
7386            runtime.forwarding.as_deref(),
7387            snapshot,
7388            &spec.module_id,
7389        )
7390        .await?;
7391        reset_restart_count(snapshot, &spec.module_id)?;
7392        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7393        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7394            Ok(next_child) => next_child,
7395            Err(err) => {
7396                if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7397                    state.state = ModuleState::Failed;
7398                    clear_current_process_facts(state);
7399                }) {
7400                    error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7401                }
7402                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7403                return Err(err);
7404            }
7405        };
7406        *child = Some(next_child);
7407        debug!(module_id = %spec.module_id, "supervised module enabled");
7408        Ok(true)
7409    } else {
7410        let stop_notice = begin_forwarding_drain_if_configured(
7411            spec,
7412            runtime,
7413            registry,
7414            snapshot,
7415            Some(false),
7416            RouteCloseReason::Disable,
7417        )
7418        .await?;
7419        drain_optional_child(
7420            &spec.module_id,
7421            spec.protocol,
7422            stop_notice,
7423            registry,
7424            runtime.forwarding.as_deref(),
7425            snapshot,
7426            &runtime.terminal_ring,
7427            &runtime.spawn_events,
7428            child,
7429            runtime.drain_timeout,
7430            ModuleState::Disabled,
7431            Some(false),
7432        )
7433        .await?;
7434        debug!(module_id = %spec.module_id, "supervised module disabled");
7435        Ok(true)
7436    }
7437}
7438
7439#[allow(clippy::too_many_arguments)]
7440async fn on_child_exit(
7441    spec: &ModuleSpec,
7442    policy: RestartPolicy,
7443    registry: &Registry,
7444    snapshot: &SharedSnapshot,
7445    terminal_ring: &Arc<Mutex<TerminalRing>>,
7446    spawn_events: &SpawnEventFeed,
7447    roster: &ChildRoster,
7448    exit_report: ExitReport,
7449) -> NextAction {
7450    // Once the daemon has begun shutting down, no exit is a crash to recover
7451    // from: the module is exiting because the daemon is going away (EOF on its
7452    // connection, or a service manager signalling the whole cgroup). Record it
7453    // as such and never schedule a respawn, which would only start a process
7454    // for the shutdown to end again.
7455    if roster.is_closed() {
7456        return on_child_exit_during_daemon_shutdown(
7457            spec,
7458            registry,
7459            snapshot,
7460            terminal_ring,
7461            spawn_events,
7462            exit_report,
7463        )
7464        .await;
7465    }
7466    // Every stop the supervisor itself asks for (operator stop, disable,
7467    // restart, reload, swap, a health restart, a drain that runs out of budget)
7468    // takes the child out of the supervise loop and reaps it in
7469    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
7470    // that reaches this point was not requested by the daemon.
7471    //
7472    // For a subc-wire module a clean exit is still a stop: those modules are
7473    // written to re-raise SIGTERM, so a stray outside signal already reads as a
7474    // crash, and exiting 0 is a deliberate choice the module made. A
7475    // `protocol: "none"` module is a stock program we cannot change, and many
7476    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
7477    // stop would leave the module down for good after any stray signal, so it
7478    // goes through the crash path instead: it spends restart budget, respawns
7479    // with the crash backoff, and ends `failed` when the budget runs out.
7480    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7481        && running_protocol(spec, snapshot) == ModuleProtocol::None;
7482    match exit_report.kind {
7483        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7484            info!(
7485                module_id = %spec.module_id,
7486                exit_code = ?exit_report.code,
7487                exit_signal = ?exit_report.signal,
7488                "supervised module exited cleanly"
7489            );
7490            if let Err(err) = publish_after_terminal_record(
7491                &spec.module_id,
7492                || {
7493                    record_terminal(
7494                        &spec.module_id,
7495                        terminal_ring,
7496                        spawn_events,
7497                        &exit_report,
7498                        TerminalDisposition::Stopped,
7499                    )
7500                },
7501                || {
7502                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7503                        state.state = ModuleState::Stopped;
7504                        clear_current_process_facts(state);
7505                        state.last_exit = Some(exit_report.clone());
7506                    })
7507                },
7508            ) {
7509                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7510            }
7511            let registration_released = match wait_for_registration_release(
7512                registry,
7513                &spec.module_id,
7514                REGISTRY_RELEASE_TIMEOUT,
7515            )
7516            .await
7517            {
7518                Ok(()) => true,
7519                Err(err) => {
7520                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7521                    false
7522                }
7523            };
7524            NextAction::Stop {
7525                registration_released,
7526            }
7527        }
7528        ExitKind::Clean | ExitKind::Crash => {
7529            if unrequested_clean_exit_of_protocol_none {
7530                warn!(
7531                    module_id = %spec.module_id,
7532                    exit_code = ?exit_report.code,
7533                    exit_signal = ?exit_report.signal,
7534                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
7535                );
7536            } else {
7537                warn!(
7538                    module_id = %spec.module_id,
7539                    exit_code = ?exit_report.code,
7540                    exit_signal = ?exit_report.signal,
7541                    "supervised module exited abnormally (crash)"
7542                );
7543            }
7544            let mut restart_schedule = None;
7545            let mut disposition = TerminalDisposition::Disabled;
7546            let mut final_state = ModuleState::Disabled;
7547            // Set only when the budget is what stopped the module, so the
7548            // terminal record says which limit was hit rather than leaving
7549            // `failed` to be read as "crashed once, badly".
7550            let mut disposition_detail = lock_snapshot(snapshot)
7551                .ok()
7552                .and_then(|mut state| state.spawn_failure.take());
7553            let now = Instant::now();
7554            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7555                if state.enabled {
7556                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
7557                        // Reserving the last slot exhausts the budget for the
7558                        // replacement, not for this exit. Publish its scheduled
7559                        // recovery atomically with the reservation so connection
7560                        // cleanup sees it while terminal recording still holds
7561                        // the visible state at Running.
7562                        state.respawn_pending = true;
7563                        final_state = ModuleState::Restarting;
7564                        restart_schedule = Some(schedule);
7565                        disposition = TerminalDisposition::Restarting;
7566                    } else {
7567                        final_state = ModuleState::Failed;
7568                        disposition = TerminalDisposition::Failed;
7569                        let budget = policy.budget_exhausted_detail();
7570                        disposition_detail =
7571                            Some(disposition_detail.take().map_or_else(
7572                                || budget.clone(),
7573                                |cause| format!("{cause}; {budget}"),
7574                            ));
7575                    }
7576                } else {
7577                    disposition = TerminalDisposition::Disabled;
7578                }
7579            }) {
7580                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7581                return NextAction::Stop {
7582                    registration_released: false,
7583                };
7584            }
7585            if disposition == TerminalDisposition::Failed {
7586                // The window is in the message, not only in the fields: this line
7587                // is read in a scrollback where a bare `max_restarts=3` reads as a
7588                // lifetime cap and sends the operator looking for three crashes
7589                // that never happened together.
7590                error!(
7591                    module_id = %spec.module_id,
7592                    max_restarts = policy.max_restarts,
7593                    window_secs = policy.window.as_secs(),
7594                    "module stopped: {}",
7595                    policy.budget_exhausted_detail()
7596                );
7597            }
7598            if let Err(err) = publish_after_terminal_record(
7599                &spec.module_id,
7600                || {
7601                    record_terminal_with_detail(
7602                        &spec.module_id,
7603                        terminal_ring,
7604                        spawn_events,
7605                        &exit_report,
7606                        disposition,
7607                        disposition_detail,
7608                    )
7609                },
7610                || {
7611                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7612                        state.state = final_state;
7613                        clear_current_process_facts(state);
7614                        state.last_exit = Some(exit_report.clone());
7615                    })
7616                },
7617            ) {
7618                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7619            }
7620
7621            if let Some(schedule) = restart_schedule {
7622                NextAction::Restart {
7623                    schedule: Some(schedule),
7624                }
7625            } else {
7626                let registration_released = match wait_for_registration_release(
7627                    registry,
7628                    &spec.module_id,
7629                    REGISTRY_RELEASE_TIMEOUT,
7630                )
7631                .await
7632                {
7633                    Ok(()) => true,
7634                    Err(err) => {
7635                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7636                        false
7637                    }
7638                };
7639                NextAction::Stop {
7640                    registration_released,
7641                }
7642            }
7643        }
7644        ExitKind::DeliberateSeverance => {
7645            warn!(
7646                module_id = %spec.module_id,
7647                exit_code = ?exit_report.code,
7648                exit_signal = ?exit_report.signal,
7649                "supervised module exited after deliberate connection severance"
7650            );
7651            let should_restart = match lock_snapshot(snapshot) {
7652                Ok(state) => state.enabled,
7653                Err(err) => {
7654                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
7655                    return NextAction::Stop {
7656                        registration_released: false,
7657                    };
7658                }
7659            };
7660            let final_state = if should_restart {
7661                ModuleState::Restarting
7662            } else {
7663                ModuleState::Disabled
7664            };
7665            if let Err(err) = publish_after_terminal_record(
7666                &spec.module_id,
7667                || {
7668                    record_terminal(
7669                        &spec.module_id,
7670                        terminal_ring,
7671                        spawn_events,
7672                        &exit_report,
7673                        terminal_disposition(final_state),
7674                    )
7675                },
7676                || {
7677                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7678                        state.state = final_state;
7679                        clear_current_process_facts(state);
7680                        state.last_exit = Some(exit_report.clone());
7681                        state.lifetime_restarts += 1;
7682                    })
7683                },
7684            ) {
7685                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
7686                return NextAction::Stop {
7687                    registration_released: false,
7688                };
7689            }
7690
7691            if should_restart {
7692                NextAction::Restart { schedule: None }
7693            } else {
7694                let registration_released = match wait_for_registration_release(
7695                    registry,
7696                    &spec.module_id,
7697                    REGISTRY_RELEASE_TIMEOUT,
7698                )
7699                .await
7700                {
7701                    Ok(()) => true,
7702                    Err(err) => {
7703                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
7704                        false
7705                    }
7706                };
7707                NextAction::Stop {
7708                    registration_released,
7709                }
7710            }
7711        }
7712    }
7713}
7714
7715async fn on_child_exit_during_daemon_shutdown(
7716    spec: &ModuleSpec,
7717    registry: &Registry,
7718    snapshot: &SharedSnapshot,
7719    terminal_ring: &Arc<Mutex<TerminalRing>>,
7720    spawn_events: &SpawnEventFeed,
7721    exit_report: ExitReport,
7722) -> NextAction {
7723    info!(
7724        module_id = %spec.module_id,
7725        exit_code = ?exit_report.code,
7726        exit_signal = ?exit_report.signal,
7727        exit_kind = ?exit_report.kind,
7728        "supervised module exited during daemon shutdown; not restarting it"
7729    );
7730    if let Err(err) = publish_after_terminal_record(
7731        &spec.module_id,
7732        || {
7733            record_terminal(
7734                &spec.module_id,
7735                terminal_ring,
7736                spawn_events,
7737                &exit_report,
7738                TerminalDisposition::DaemonShutdown,
7739            )
7740        },
7741        || {
7742            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7743                state.state = ModuleState::Stopped;
7744                clear_current_process_facts(state);
7745                state.last_exit = Some(exit_report.clone());
7746            })
7747        },
7748    ) {
7749        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
7750    }
7751    let registration_released =
7752        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
7753            .await
7754            .is_ok();
7755    NextAction::Stop {
7756        registration_released,
7757    }
7758}
7759
7760// Keep the status lock out of terminal I/O. Readers may observe the previous
7761// state while recording is in progress, but never a completed exit without its
7762// history. Journal errors are logged by the journal and retained in the ring;
7763// a recording panic still publishes the final state before resuming the unwind.
7764fn publish_after_terminal_record<T>(
7765    module_id: &str,
7766    record: impl FnOnce(),
7767    publish: impl FnOnce() -> T,
7768) -> T {
7769    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
7770    let published = publish();
7771    if let Err(panic) = recorded {
7772        error!(
7773            module_id,
7774            "terminal recording panicked; published final state before resuming unwind"
7775        );
7776        std::panic::resume_unwind(panic);
7777    }
7778    published
7779}
7780
7781fn fail_after_wait_error(
7782    module_id: &str,
7783    snapshot: &SharedSnapshot,
7784    terminal_ring: &Arc<Mutex<TerminalRing>>,
7785    spawn_events: &SpawnEventFeed,
7786    detail: String,
7787) {
7788    let report = wait_error_exit_report();
7789    publish_after_terminal_record(
7790        module_id,
7791        || {
7792            record_terminal_with_detail(
7793                module_id,
7794                terminal_ring,
7795                spawn_events,
7796                &report,
7797                TerminalDisposition::Failed,
7798                Some(detail),
7799            )
7800        },
7801        || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
7802    );
7803}
7804
7805#[cfg(test)]
7806fn record_wait_error_terminal(
7807    module_id: &str,
7808    terminal_ring: &Arc<Mutex<TerminalRing>>,
7809    spawn_events: &SpawnEventFeed,
7810) {
7811    record_terminal(
7812        module_id,
7813        terminal_ring,
7814        spawn_events,
7815        &wait_error_exit_report(),
7816        TerminalDisposition::Failed,
7817    );
7818}
7819
7820fn record_terminal(
7821    module_id: &str,
7822    terminal_ring: &Arc<Mutex<TerminalRing>>,
7823    spawn_events: &SpawnEventFeed,
7824    exit_report: &ExitReport,
7825    disposition: TerminalDisposition,
7826) {
7827    record_terminal_with_detail(
7828        module_id,
7829        terminal_ring,
7830        spawn_events,
7831        exit_report,
7832        disposition,
7833        None,
7834    );
7835}
7836
7837/// The ring lock is held only to capture the read (see
7838/// `TerminalJournal::capture_read`), so this module's exits keep recording
7839/// while the journal files are read. Blocking: it reads files.
7840fn durable_terminal_history_of(
7841    terminal_ring: &Mutex<TerminalRing>,
7842    module_id: &str,
7843) -> subc_control::TerminalHistory {
7844    let read = terminal_ring
7845        .lock()
7846        .unwrap_or_else(|p| p.into_inner())
7847        .capture_durable_history();
7848    read.read(module_id)
7849}
7850
7851fn record_terminal_with_detail(
7852    module_id: &str,
7853    terminal_ring: &Arc<Mutex<TerminalRing>>,
7854    spawn_events: &SpawnEventFeed,
7855    exit_report: &ExitReport,
7856    disposition: TerminalDisposition,
7857    disposition_detail: Option<String>,
7858) {
7859    #[cfg(test)]
7860    terminal_history_tests::before_terminal_record(spawn_events);
7861    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
7862    let record = TerminalRecord {
7863        exit_code: exit_report.code,
7864        exit_signal: exit_report.signal,
7865        at_ms: exit_report.at_ms,
7866        disposition,
7867        exit_kind: exit_report.kind.into(),
7868        disposition_detail,
7869    };
7870    terminal_ring
7871        .lock()
7872        .unwrap_or_else(|poisoned| poisoned.into_inner())
7873        .record_exit(module_id, record);
7874}
7875
7876fn untrack_if_registration_released(
7877    process_liveness: &SupervisorProcessLiveness,
7878    registry: &Registry,
7879    module_id: &str,
7880    snapshot: &SharedSnapshot,
7881) {
7882    match registry.get_module(module_id) {
7883        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
7884        Ok(Some(_)) => {}
7885        Err(err) => {
7886            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
7887        }
7888    }
7889}
7890
7891/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
7892/// then apply the module's configured entries minus daemon-private capture keys.
7893///
7894/// Separated from `spawn_child` only so it can be asserted without spawning a
7895/// process — a duplicate of this logic in a test would pass while the real one
7896/// drifted, which is the defect class this function exists to avoid.
7897/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
7898/// nonce. A `protocol: "none"` module gets neither, because it cannot use
7899/// either and the argument would stop a stock binary from starting at all.
7900/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
7901///
7902/// The plain-spawn form, kept for the tests that assert its plan; spawns go
7903/// through [`apply_wire_spawn_args_for_role`].
7904#[cfg(test)]
7905fn apply_wire_spawn_args(
7906    command: &mut Command,
7907    spec: &ModuleSpec,
7908    connection_file_path: Option<&std::path::Path>,
7909    handle: Option<&SupervisorHandle>,
7910) -> Result<Option<NonceHandoff>, SuperviseError> {
7911    apply_wire_spawn_args_for_role(
7912        command,
7913        spec,
7914        connection_file_path,
7915        handle,
7916        SpawnRole::Plain,
7917    )
7918}
7919
7920/// The read end of a spawn's launch-nonce pipe, prepared by
7921/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
7922/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
7923/// server before spawn and authenticates its client after the child exists.
7924#[cfg(unix)]
7925type NonceHandoff = subc_os::LaunchNonceHandoff;
7926#[cfg(windows)]
7927type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
7928#[cfg(not(any(unix, windows)))]
7929type NonceHandoff = std::convert::Infallible;
7930
7931/// Create the launch nonce that a new module process must present when it
7932/// registers or opens a consumer connection, including a replacement process
7933/// started alongside a still-serving module.
7934///
7935/// A plain spawn replaces the module's recorded nonce. A swap candidate records
7936/// a separate candidate token so the still-serving incumbent and its consumers
7937/// keep their nonce. Both records are installed before the process exists, so
7938/// the child's initial HELLO registration cannot arrive ahead of its nonce.
7939///
7940/// On Unix the nonce is delivered only through a pipe. It is written into
7941/// a pipe whose read end the child gets as descriptor 3, named by
7942/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
7943/// process of the same user cannot read it with `ps eww`. That handoff is
7944/// returned rather than installed here, because installing it replaces
7945/// whatever the child has at descriptor 3 and so must be the last pre-exec
7946/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
7947/// the Linux callback that writes the process into its cgroup. Only after that
7948/// does `handoff.install_last` register the descriptor-3 replacement, so it
7949/// cannot close a descriptor the cgroup callback still needs.
7950/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
7951fn apply_wire_spawn_args_for_role(
7952    command: &mut Command,
7953    spec: &ModuleSpec,
7954    connection_file_path: Option<&std::path::Path>,
7955    handle: Option<&SupervisorHandle>,
7956    role: SpawnRole,
7957) -> Result<Option<NonceHandoff>, SuperviseError> {
7958    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
7959    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
7960    // included: a daemon started from a module's process tree inherits it,
7961    // and passing it on would point the child at a descriptor it does not
7962    // have.
7963    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
7964    #[cfg(windows)]
7965    {
7966        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
7967        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
7968    }
7969    // Remove inherited or configured copies too: withholding must mean absent.
7970    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
7971    if spec.protocol == ModuleProtocol::None {
7972        return Ok(None);
7973    }
7974    if let Some(connection_file_path) = connection_file_path {
7975        command.arg(SUBC_ARG).arg(connection_file_path);
7976    }
7977
7978    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
7979    // route.open attestation. Reserved modules additionally use the same nonce
7980    // for HELLO id-squatting protection. A respawn rotates both records.
7981    let nonce = generate_launch_nonce()?;
7982    if let Some(handle) = handle {
7983        match role {
7984            SpawnRole::Plain => {
7985                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
7986                if spec.reserved {
7987                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
7988                }
7989            }
7990            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
7991        }
7992    }
7993    #[cfg(unix)]
7994    let handoff = {
7995        let handoff =
7996            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
7997                program: spec.program.clone(),
7998                source,
7999                cgroup_path: None,
8000            })?;
8001        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8002        Some(handoff)
8003    };
8004    #[cfg(windows)]
8005    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8006    #[cfg(not(any(unix, windows)))]
8007    let handoff = None;
8008    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
8009    // until the daemon's per-spawn delivery records show that every running
8010    // Windows module has read its secret from the named pipe.
8011    #[cfg(not(unix))]
8012    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8013    Ok(handoff)
8014}
8015
8016fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8017    command.env_remove(CK_LOG_ENV);
8018    // The spawn role is the supervisor's to set, and only on a swap candidate
8019    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8020    // it, is what makes it absent on a plain spawn: the daemon's own
8021    // environment could carry it, and so could a spec built outside daemon
8022    // config (config refuses it as an `env` key). A module reading it on a
8023    // plain restart would pick the long swap budget and leave callers waiting.
8024    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8025    for (key, value) in &spec.env {
8026        // cortexkit-log currently exposes retention only as a Rust struct, not
8027        // environment names. These values are daemon-private sink metadata and
8028        // must never become a public child-process contract by being inherited.
8029        if matches!(
8030            key.as_str(),
8031            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8032        ) || key == SUBC_SPAWN_ROLE_ENV
8033        {
8034            continue;
8035        }
8036        command.env(key, value);
8037    }
8038}
8039
8040#[cfg(windows)]
8041fn prepare_windows_nonce_pipe(
8042    command: &mut Command,
8043    spec: &ModuleSpec,
8044    nonce: &str,
8045) -> Option<NonceHandoff> {
8046    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8047        Ok(handoff) => {
8048            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8049            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8050            // still register until every module reads the pipe. A reader without
8051            // this variable set to env returns the pipe error instead of using
8052            // an inherited or configured environment copy.
8053            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8054            Some(handoff)
8055        }
8056        Err(error) => {
8057            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8058            None
8059        }
8060    }
8061}
8062
8063#[cfg(windows)]
8064fn start_windows_nonce_pipe(
8065    handoff: Option<NonceHandoff>,
8066    pid: u32,
8067    spec: &ModuleSpec,
8068) -> Option<subc_os::LaunchNoncePipeDelivery> {
8069    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8070    // same maximum wait used for registration during a reload. Older modules
8071    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8072    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8073        Ok(delivery) => Some(delivery),
8074        Err(error) => {
8075            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8076            None
8077        }
8078    })
8079}
8080
8081fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8082    #[cfg(windows)]
8083    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8084        return Some(
8085            if snapshot
8086                .pipe_consumed
8087                .as_ref()
8088                .is_some_and(|consumed| consumed.get().is_some())
8089            {
8090                "pipe"
8091            } else {
8092                "env"
8093            }
8094            .to_owned(),
8095        );
8096    }
8097    let _ = snapshot;
8098    None
8099}
8100
8101/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8102/// of a blue/green swap.
8103#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8104enum SpawnRole {
8105    Plain,
8106    SwapCandidate,
8107}
8108
8109/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8110/// `apply_child_env` has already removed the variable for every spawn.
8111fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8112    if role == SpawnRole::SwapCandidate {
8113        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8114    }
8115}
8116
8117fn previous_instance_error(
8118    spec: &ModuleSpec,
8119    pid: u32,
8120    reason: crate::live_children::RetainedReason,
8121) -> SuperviseError {
8122    SuperviseError::Spawn {
8123        program: spec.program.clone(),
8124        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8125        cgroup_path: None,
8126    }
8127}
8128
8129fn spawn_child(
8130    spec: &ModuleSpec,
8131    connection_file_path: Option<&std::path::Path>,
8132    handle: Option<&SupervisorHandle>,
8133    ring: &Arc<Mutex<StderrRing>>,
8134    capture_logs_dir: Option<&std::path::Path>,
8135    roster: &ChildRoster,
8136    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8137) -> Result<SupervisedChild, SuperviseError> {
8138    spawn_child_in_slot(
8139        spec,
8140        connection_file_path,
8141        handle,
8142        ring,
8143        capture_logs_dir,
8144        roster,
8145        #[cfg(target_os = "linux")]
8146        cgroup_placement,
8147        SpawnRole::Plain,
8148        false,
8149    )
8150}
8151
8152/// Spawn one process of `spec` into a slot.
8153///
8154/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
8155/// A swap candidate needs a different cgroup from the process it is replacing,
8156/// which is still alive: in the same cgroup the two would be one kill domain,
8157/// and killing a failed candidate could take the incumbent with it.
8158///
8159/// The stderr capture file is `<module_id>.stderr.log` for every process of
8160/// the module, whichever slot it is in, because that is the one file
8161/// `ck module logs` reads. During a swap's overlap both processes append to it;
8162/// the daemon writes whole lines, so the two interleave by line, which is also
8163/// the merged view an operator wants while a swap runs.
8164#[allow(clippy::too_many_arguments)]
8165fn spawn_child_in_slot(
8166    spec: &ModuleSpec,
8167    connection_file_path: Option<&std::path::Path>,
8168    handle: Option<&SupervisorHandle>,
8169    ring: &Arc<Mutex<StderrRing>>,
8170    capture_logs_dir: Option<&std::path::Path>,
8171    roster: &ChildRoster,
8172    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8173    role: SpawnRole,
8174    alternate_slot: bool,
8175) -> Result<SupervisedChild, SuperviseError> {
8176    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8177        return Err(previous_instance_error(spec, pid, reason));
8178    }
8179    if roster.is_closed() {
8180        return Err(SuperviseError::Spawn {
8181            program: spec.program.clone(),
8182            source: io::Error::other("the daemon is shutting down; not starting a new process"),
8183            cgroup_path: None,
8184        });
8185    }
8186    #[cfg(target_os = "linux")]
8187    let cgroup_name = {
8188        // Slot names alone are not kill domains: a retired incumbent may still
8189        // be draining when a later enable/restart spawns into the same slot.
8190        // Decimal entropy keeps the suffix unambiguous; Placement performs
8191        // the module-id escaping and constructs the filesystem path.
8192        if cgroup_placement.is_none() {
8193            swap::cgroup_name(&spec.module_id, alternate_slot)
8194        } else {
8195            let nonce = generate_launch_nonce()?;
8196            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8197            // Leave room for byte escaping and the suffix under NAME_MAX. The
8198            // label is only for humans; the nonce identifies the kill domain.
8199            let mut end = spec.module_id.len().min(64);
8200            while !spec.module_id.is_char_boundary(end) {
8201                end -= 1;
8202            }
8203            format!(
8204                "{}_{suffix}",
8205                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8206            )
8207        }
8208    };
8209    #[cfg(not(target_os = "linux"))]
8210    let _ = alternate_slot;
8211    #[cfg(target_os = "macos")]
8212    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8213    #[cfg(windows)]
8214    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8215        program: spec.program.clone(),
8216        source,
8217        cgroup_path: None,
8218    })?;
8219    #[cfg(windows)]
8220    let mut command = Command::new(image_capture.path());
8221    #[cfg(not(any(target_os = "macos", windows)))]
8222    let mut command = Command::new(&spec.program);
8223    command.args(&spec.args);
8224    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
8225    // that is the whole of the intent, so remove that one key rather than the
8226    // environment.
8227    //
8228    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
8229    // and took the POSIX environment with it. Modules spawned that way had no
8230    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
8231    // logging:
8232    //
8233    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
8234    //     both unset it fell back to the temp dir alone and `ck` could not find
8235    //     a daemon running on the same machine from inside any module's process
8236    //     tree — reporting a path the file has never lived at, which reads as
8237    //     "the daemon did not write its file".
8238    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
8239    //     the RELATIVE `.local/share`, so a module deriving its own store path
8240    //     resolved it against its own CWD. That is the store-fragmentation
8241    //     defect the daemon already refuses in config (`parse_doc` rejects a
8242    //     relative `storage.data_home`) arriving by derivation instead.
8243    //   * anything a module spawns inherited it: git without ~/.gitconfig,
8244    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
8245    //     quietly rather than erroring.
8246    //
8247    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
8248    // offered one candidate under /tmp while the file sat in /run/user/1000.
8249    //
8250    // A configured module is unaffected either way: `module_spec()` puts the
8251    // resolved CK_LOG into `spec.env`, which is applied below and therefore
8252    // wins over anything ambient.
8253    apply_child_env(&mut command, spec);
8254    apply_spawn_role(&mut command, role);
8255    let nonce_handoff =
8256        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8257
8258    #[cfg(target_os = "linux")]
8259    let cgroup_path = cgroup_placement
8260        .map(|placement| placement.module_path(&cgroup_name))
8261        .transpose()
8262        .map_err(|source| SuperviseError::Cgroup {
8263            module_id: spec.module_id.clone(),
8264            source,
8265        })?;
8266    #[cfg(not(target_os = "linux"))]
8267    let cgroup_path: Option<PathBuf> = None;
8268    #[cfg(target_os = "linux")]
8269    if let Some(path) = &cgroup_path {
8270        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8271            if let Some(placement) = cgroup_placement {
8272                remove_module_cgroup(placement, &cgroup_name);
8273            }
8274            return Err(error);
8275        }
8276    }
8277
8278    let output_sink = if let Some(logs_dir) = capture_logs_dir {
8279        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8280        match ChildOutputSink::open(&path, capture_retention(spec)) {
8281            Ok(sink) => sink,
8282            Err(error) => {
8283                warn!(
8284                    module_id = %spec.module_id,
8285                    path = %path.display(),
8286                    error = %error,
8287                    "could not open child output capture file; forwarding to stderr"
8288                );
8289                ChildOutputSink::Stderr
8290            }
8291        }
8292    } else {
8293        ChildOutputSink::Stderr
8294    };
8295
8296    command.stdout(Stdio::piped());
8297    command.stderr(Stdio::piped());
8298    command.kill_on_drop(true);
8299    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
8300    // before exec). In the daemon's group, a service manager that kills the
8301    // job's process group when the daemon exits (launchd's default) killed
8302    // every module at the same moment its control connection closed, so no
8303    // module ever ran its EOF teardown on a daemon stop. Outside that group a
8304    // module is reached only by the daemon: the EOF it sees when its
8305    // connection closes, and the bounded stop in `child_roster` for anything
8306    // still running after that. On Linux this composes with the cgroup
8307    // placement above: that is a pre_exec write to cgroup.procs, std performs
8308    // setpgid in the child before running pre_exec callbacks, and the two
8309    // change independent process attributes.
8310    //
8311    // stdin is /dev/null because a process outside the terminal's foreground
8312    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
8313    // by hand would otherwise hand down. Under a service manager stdin is
8314    // already /dev/null.
8315    #[cfg(unix)]
8316    command.process_group(0);
8317    command.stdin(Stdio::null());
8318    // The LAST pre-exec step, after the cgroup placement above: installing the
8319    // nonce at descriptor 3 replaces whatever the child had there, which could
8320    // be the descriptor an earlier step writes through.
8321    #[cfg(unix)]
8322    if let Some(handoff) = nonce_handoff {
8323        handoff.install_last(command.as_std_mut());
8324    }
8325    #[cfg(not(any(unix, windows)))]
8326    let _ = nonce_handoff;
8327
8328    // Containment, step 1 of 3 (issue #109): create the child suspended so it
8329    // cannot run a single instruction -- and therefore cannot spawn a
8330    // grandchild -- before it is in the job. See `contain_spawned_child` for the
8331    // other two steps and why the window matters.
8332    #[cfg(windows)]
8333    {
8334        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8335            program: spec.program.clone(),
8336            source,
8337            cgroup_path: None,
8338        })?;
8339        subc_jobobject::suspend_on_create_async(&mut command);
8340    }
8341    let mut child = match command.spawn() {
8342        Ok(child) => child,
8343        Err(source) => {
8344            #[cfg(target_os = "linux")]
8345            if let Some(placement) = cgroup_placement {
8346                remove_module_cgroup(placement, &cgroup_name);
8347            }
8348            return Err(SuperviseError::Spawn {
8349                program: spec.program.clone(),
8350                source,
8351                cgroup_path,
8352            });
8353        }
8354    };
8355    #[cfg(test)]
8356    spawn_record_tests::after_create(&child, spec);
8357    // The parent must close its writer now: the acknowledgement pipe reports EOF
8358    // only when every writer is gone, and the child's copy closes when the
8359    // trampoline replaces itself with the module. Command holds only an integer
8360    // in its pre_exec callback, not another writer.
8361    #[cfg(target_os = "macos")]
8362    drop(exec_ack);
8363
8364    // Bind while the child is still suspended, before job assignment resumes it.
8365    // Keeping the executable handle open across CreateProcessW denies rename
8366    // and deletion, so the file cannot be exchanged during spawn. Bind that
8367    // file ID to the process handle returned by spawn, never a reopened PID.
8368    #[cfg(windows)]
8369    let windows_image = match image_capture.bind_tokio(&child) {
8370        Ok(image) => Arc::new(image),
8371        Err(source) => {
8372            kill_and_reap_rejected_child(&mut child);
8373            return Err(SuperviseError::Spawn {
8374                program: spec.program.clone(),
8375                source,
8376                cgroup_path: None,
8377            });
8378        }
8379    };
8380    // Assign the suspended child to its module job; resume only after its
8381    // process identity has been durably recorded for orphan recovery.
8382    #[cfg(windows)]
8383    let job = contain_spawned_child(&mut child, spec)?;
8384    let spawned_at_ms = unix_ms_now();
8385    #[cfg(not(windows))]
8386    let spawned_from = spec.program.clone();
8387    #[cfg(windows)]
8388    let spawned_from = windows_image.path().to_path_buf();
8389    #[cfg(not(windows))]
8390    let spawned_file_identity = spawned_file_identity(&spawned_from);
8391    #[cfg(windows)]
8392    let spawned_file_identity = Some(SpawnedFileIdentity {
8393        device: 0,
8394        inode: 0,
8395        windows_image: Some(Arc::clone(&windows_image)),
8396    });
8397    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8398        program: spec.program.clone(),
8399        source: io::Error::other("spawned child exposed no live pid"),
8400        cgroup_path: cgroup_path.clone(),
8401    })?;
8402    #[cfg(windows)]
8403    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8404    #[cfg(not(windows))]
8405    let process_start_time = crate::provenance::process_start_time(pid);
8406    #[cfg(windows)]
8407    let process_start_time = Some(windows_image.start_time());
8408    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8409    #[cfg(all(test, target_os = "macos"))]
8410    privacy_exec_boundary_tests::before_image_sample(spec, pid);
8411    // Record the committed kernel image, not the configured pathname: PATH
8412    // lookup and shebang interpretation can select a different executable.
8413    // Linux's posix_spawn can return before that image is installed, so the
8414    // observer also waits for the kernel's first-exec flag to clear.
8415    #[cfg(not(windows))]
8416    let recorded_image = observe_spawned_image(pid);
8417    // spawn() confirms only the first exec, into the trampoline. Never persist
8418    // the trampoline image; the asynchronous acknowledgement publishes the
8419    // module image once the trampoline has replaced itself with the module.
8420    #[cfg(target_os = "macos")]
8421    let recorded_image = if privacy_exec.is_some() {
8422        None
8423    } else {
8424        recorded_image
8425    };
8426    #[cfg(target_os = "linux")]
8427    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8428    #[cfg(not(target_os = "linux"))]
8429    let recorded_cgroup_name = None;
8430    let roster_guard = roster.admit(
8431        spec.module_id.clone(),
8432        pid,
8433        spec.protocol,
8434        process_start_time,
8435        crate::child_roster::RecordedIdentity {
8436            #[cfg(not(windows))]
8437            start_time: recorded_image.map(|image| image.start_time),
8438            #[cfg(windows)]
8439            start_time: process_start_time,
8440            #[cfg(not(windows))]
8441            executable: recorded_image
8442                .and_then(|image| image.executable)
8443                .map(crate::live_children::ExecutableIdentity::from),
8444            #[cfg(windows)]
8445            executable: Some(crate::live_children::ExecutableIdentity::from(
8446                windows_image.identity(),
8447            )),
8448            cgroup_name: recorded_cgroup_name,
8449            #[cfg(target_os = "linux")]
8450            cgroup_placement: cgroup_placement.cloned(),
8451        },
8452    );
8453    let roster_guard = match roster_guard {
8454        Ok(guard) => guard,
8455        Err(source) => {
8456            #[cfg(windows)]
8457            if let Some(job) = &job {
8458                let _ = job.terminate();
8459            }
8460            #[cfg(target_os = "linux")]
8461            kill_module_cgroup(cgroup_placement, &cgroup_name);
8462            kill_and_reap_rejected_child(&mut child);
8463            #[cfg(target_os = "linux")]
8464            if let Some(placement) = cgroup_placement {
8465                remove_module_cgroup(placement, &cgroup_name);
8466            }
8467            return Err(SuperviseError::Spawn {
8468                program: spec.program.clone(),
8469                source: io::Error::other(format!(
8470                    "could not durably record module child: {source}"
8471                )),
8472                cgroup_path,
8473            });
8474        }
8475    };
8476    // The check at the top of this function can pass just before daemon
8477    // shutdown begins, and the process is only in the roster from here on.
8478    // The shutdown stop returns as soon as it finds the roster empty, so a
8479    // process admitted after that look would outlive the daemon. The roster
8480    // is closed before the stop first reads it and admission happens under
8481    // the roster's lock, so either the stop sees this process or this check
8482    // sees the roster closed: end the process now rather than start a module
8483    // the daemon is about to stop.
8484    if roster.is_closed() {
8485        // This child was never admitted, so there is no module protocol shutdown to wait for.
8486        #[cfg(target_os = "linux")]
8487        kill_module_cgroup(cgroup_placement, &cgroup_name);
8488        if let Err(error) = child.start_kill() {
8489            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8490        }
8491        #[cfg(target_os = "linux")]
8492        if let Some(placement) = cgroup_placement {
8493            // This spawn was never admitted, so shutdown has no roster entry
8494            // to await. Do not detach its cleanup: the runtime could exit
8495            // before that task reaps the rejected child and removes its group.
8496            while matches!(child.try_wait(), Ok(None)) {
8497                std::thread::yield_now();
8498            }
8499            if matches!(
8500                subc_cgroup::kill_module(Some(placement), &cgroup_name),
8501                subc_cgroup::KillOutcome::Killed
8502            ) {
8503                if let Ok(path) = placement.module_path(&cgroup_name) {
8504                    while std::fs::read_to_string(path.join("cgroup.events"))
8505                        .ok()
8506                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8507                    {
8508                        std::thread::yield_now();
8509                    }
8510                }
8511            }
8512            remove_module_cgroup(placement, &cgroup_name);
8513        }
8514        drop(roster_guard);
8515        return Err(SuperviseError::Spawn {
8516            program: spec.program.clone(),
8517            source: io::Error::other(
8518                "the daemon began shutting down while this process was starting; ended it",
8519            ),
8520            cgroup_path,
8521        });
8522    }
8523
8524    #[cfg(windows)]
8525    if let Err(error) = resume_suspended_child(pid, spec) {
8526        kill_and_reap_rejected_child(&mut child);
8527        return Err(error);
8528    }
8529
8530    let stdout_pump = match child.stdout.take() {
8531        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8532        None => {
8533            warn!(
8534                module_id = %spec.module_id,
8535                "spawned child exposed no stdout pipe; file capture will be incomplete"
8536            );
8537            None
8538        }
8539    };
8540    let stderr_pump = match child.stderr.take() {
8541        Some(stderr) => {
8542            let generation = ring
8543                .lock()
8544                .unwrap_or_else(|poisoned| poisoned.into_inner())
8545                .begin_process();
8546            Some(StderrPump {
8547                task: tokio::spawn(pump_stderr_to(
8548                    stderr,
8549                    Arc::clone(ring),
8550                    generation,
8551                    output_sink,
8552                )),
8553                generation,
8554            })
8555        }
8556        None => {
8557            // Spawning succeeded but the pipe did not materialise. Recording it as
8558            // uncaptured keeps the tail honest: the alternative is an empty tail
8559            // that reads as a module which printed nothing.
8560            ring.lock()
8561                .unwrap_or_else(|poisoned| poisoned.into_inner())
8562                .mark_not_captured("stderr pipe was not available on spawn");
8563            warn!(
8564                module_id = %spec.module_id,
8565                "spawned child exposed no stderr pipe; tail will be unavailable"
8566            );
8567            None
8568        }
8569    };
8570
8571    let exec = Arc::new(crate::activity::ExecOutcome::new());
8572    #[cfg(target_os = "macos")]
8573    let pending_exec = privacy_exec.is_some();
8574    #[cfg(not(target_os = "macos"))]
8575    let pending_exec = false;
8576    if !pending_exec {
8577        exec.finish(crate::activity::ExecState::CommittedExec);
8578    }
8579    Ok(SupervisedChild {
8580        child,
8581        exec: crate::activity::ExecOwner(exec),
8582        #[cfg(windows)]
8583        nonce_delivery,
8584        protocol: spec.protocol,
8585        #[cfg(target_os = "linux")]
8586        module_id: cgroup_name,
8587        #[cfg(target_os = "linux")]
8588        cgroup_placement: cgroup_placement.cloned(),
8589        #[cfg(windows)]
8590        job,
8591        stdout_pump,
8592        stderr_pump,
8593        stderr_ring: Arc::clone(ring),
8594        spawned_at_ms,
8595        spawned_from,
8596        spawned_file_identity,
8597        process_start_time,
8598        process_identity,
8599        pid,
8600        roster_guard: Some(roster_guard),
8601        #[cfg(target_os = "macos")]
8602        privacy_exec,
8603        #[cfg(target_os = "macos")]
8604        report_ready: Arc::new(OnceLock::new()),
8605        spawn_failure: None,
8606    })
8607}
8608
8609#[cfg(test)]
8610mod spawn_record_tests {
8611    use super::*;
8612
8613    thread_local! {
8614        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8615    }
8616
8617    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8618        LAST_PID.set(child.id());
8619        #[cfg(windows)]
8620        if let Some((_, address)) = _spec
8621            .env
8622            .iter()
8623            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8624        {
8625            use std::io::{Read, Write};
8626            let mut stream = std::net::TcpStream::connect(address).unwrap();
8627            stream
8628                .write_all(&child.id().unwrap().to_ne_bytes())
8629                .unwrap();
8630            let mut release = [0];
8631            stream.read_exact(&mut release).unwrap();
8632        }
8633    }
8634
8635    #[cfg(windows)]
8636    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8637        if spec
8638            .env
8639            .iter()
8640            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8641        {
8642            Err(io::Error::other("injected module job assignment failure"))
8643        } else {
8644            Ok(())
8645        }
8646    }
8647
8648    fn spec() -> ModuleSpec {
8649        ModuleSpec {
8650            module_id: "spawn-record-worker".into(),
8651            #[cfg(unix)]
8652            program: "/bin/sleep".into(),
8653            #[cfg(windows)]
8654            program: std::env::current_exe().unwrap(),
8655            #[cfg(unix)]
8656            args: vec!["60".into()],
8657            #[cfg(windows)]
8658            args: vec![
8659                "--exact".into(),
8660                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
8661                "--nocapture".into(),
8662            ],
8663            env: vec![],
8664            reserved: false,
8665            reserved_prefixes: vec![],
8666            protocol: ModuleProtocol::None,
8667            overlap: Default::default(),
8668        }
8669    }
8670
8671    #[tokio::test]
8672    async fn record_sync_failure_fails_spawn_and_reaps_child() {
8673        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
8674        let supervisor =
8675            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8676                .with_live_children_record(home.join("live-children.json"));
8677        crate::live_children::fail_next_file_sync();
8678        let result = supervisor.spawn(spec());
8679        let failed = result.is_err();
8680        if let Ok(module) = result {
8681            module.stop().await.unwrap();
8682        }
8683        assert!(failed, "record failure must fail spawn");
8684        let pid = LAST_PID.get().expect("the real OS child was created");
8685        assert!(
8686            subc_os::Process::open(pid).unwrap().is_none(),
8687            "child {pid} survived rejected admission or was not reaped"
8688        );
8689    }
8690
8691    #[cfg(target_os = "macos")]
8692    #[test]
8693    #[ignore = "manual on-device latency measurement"]
8694    fn durable_record_write_latency_50() {
8695        let home = cortexkit_test_support::ScratchDir::new("record-latency");
8696        let path = home.join("live-children.json");
8697        let children = vec![crate::live_children::LiveChild {
8698            module_id: "latency-worker".into(),
8699            pid: std::process::id(),
8700            protocol: ModuleProtocol::None,
8701            start_time: Some(1),
8702            executable: None,
8703            cgroup_name: None,
8704        }];
8705        let mut samples = Vec::new();
8706        for _ in 0..50 {
8707            let start = std::time::Instant::now();
8708            crate::live_children::write_record(&path, &children).unwrap();
8709            samples.push(start.elapsed());
8710        }
8711        samples.sort();
8712        eprintln!(
8713            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
8714            (samples[24] + samples[25]) / 2,
8715            samples[47]
8716        );
8717    }
8718
8719    #[cfg(windows)]
8720    #[test]
8721    fn windows_store_touch_fixture() {
8722        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
8723            std::fs::write(path, b"module ran").unwrap();
8724        }
8725    }
8726
8727    #[cfg(windows)]
8728    #[tokio::test]
8729    async fn windows_daemon_fixture() {
8730        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
8731            return;
8732        };
8733        let mut spec = spec();
8734        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
8735        let supervisor =
8736            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8737        let _module = supervisor.spawn(spec).unwrap();
8738    }
8739
8740    #[cfg(windows)]
8741    #[test]
8742    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
8743        use std::io::Read;
8744        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
8745        // An outer job models a scheduler/service host. The daemon's root and
8746        // root job must nest without requiring breakaway from this job.
8747        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8748        command
8749            .args([
8750                "--exact",
8751                "supervise::spawn_record_tests::windows_daemon_fixture",
8752                "--nocapture",
8753            ])
8754            .env(
8755                "SUBC_TEST_DAEMON_BARRIER",
8756                listener.local_addr().unwrap().to_string(),
8757            );
8758        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8759        listener.set_nonblocking(true).unwrap();
8760        let deadline = std::time::Instant::now() + Duration::from_secs(30);
8761        let mut stream = loop {
8762            if let Ok((stream, _)) = listener.accept() {
8763                break stream;
8764            }
8765            assert!(
8766                daemon.child.try_wait().unwrap().is_none(),
8767                "daemon failed before rendezvous"
8768            );
8769            assert!(
8770                std::time::Instant::now() < deadline,
8771                "daemon did not reach CreateProcess failpoint"
8772            );
8773            std::thread::yield_now();
8774        };
8775        stream
8776            .set_read_timeout(Some(Duration::from_secs(30)))
8777            .unwrap();
8778        let mut pid = [0; 4];
8779        stream.read_exact(&mut pid).unwrap();
8780        let pid = u32::from_ne_bytes(pid);
8781        assert!(subc_jobobject::process_exists(pid));
8782        // Terminate only the daemon, not its outer job: otherwise the host job
8783        // would also kill a child whose daemon never joined its root job.
8784        daemon.child.kill().unwrap();
8785        daemon.child.wait().unwrap();
8786        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
8787        if !exited {
8788            daemon.job.terminate().unwrap();
8789        }
8790        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
8791    }
8792
8793    #[cfg(windows)]
8794    #[tokio::test]
8795    async fn windows_containment_failure_never_resumes() {
8796        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
8797        let sentinel = home.join("store-touched");
8798        let mut spec = spec();
8799        spec.env
8800            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
8801        spec.env.push((
8802            "SUBC_TEST_STORE_TOUCH".into(),
8803            sentinel.display().to_string(),
8804        ));
8805        let supervisor =
8806            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8807        let result = supervisor.spawn(spec);
8808        // A version that resumes despite assignment failure must have time to
8809        // execute its sentinel before dropping the returned module kills it.
8810        if let Ok(module) = result {
8811            let _ = tokio::time::timeout(Duration::from_secs(10), async {
8812                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
8813                    tokio::task::yield_now().await;
8814                }
8815            })
8816            .await;
8817            let _ = module.stop().await;
8818            panic!(
8819                "assignment failure resumed a module; store touched={}",
8820                sentinel.exists()
8821            );
8822        }
8823        assert!(
8824            !sentinel.exists(),
8825            "refused module executed store-touch sentinel"
8826        );
8827        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
8828    }
8829
8830    #[cfg(windows)]
8831    #[tokio::test]
8832    async fn windows_nested_daemon_fixture() {
8833        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
8834            return;
8835        };
8836        let mut spec = spec();
8837        spec.env.push((
8838            "SUBC_TEST_STORE_TOUCH".into(),
8839            sentinel.to_str().unwrap().into(),
8840        ));
8841        let supervisor =
8842            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8843        let module = supervisor
8844            .spawn(spec)
8845            .expect("per-module job must nest below daemon and host jobs");
8846        tokio::time::timeout(Duration::from_secs(10), async {
8847            while !PathBuf::from(&sentinel).exists() {
8848                tokio::task::yield_now().await;
8849            }
8850        })
8851        .await
8852        .unwrap();
8853        module.stop().await.unwrap();
8854    }
8855
8856    #[cfg(windows)]
8857    #[test]
8858    fn windows_module_job_nests_under_host_job() {
8859        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
8860        let sentinel = home.join("nested-store-touched");
8861        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8862        command
8863            .args([
8864                "--exact",
8865                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
8866                "--nocapture",
8867            ])
8868            .env("SUBC_TEST_NESTED_STORE", &sentinel);
8869        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8870        assert!(daemon.child.wait().unwrap().success());
8871        assert!(sentinel.exists(), "nested protocol:none module never ran");
8872    }
8873}
8874
8875#[cfg(target_os = "linux")]
8876pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
8877    use subc_cgroup::KillOutcome;
8878    match subc_cgroup::kill_module(placement, module_id) {
8879        KillOutcome::Killed => {}
8880        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
8881            debug!(
8882                module_id,
8883                "cgroup tree kill unavailable; using direct-child kill"
8884            );
8885        }
8886        KillOutcome::IoError { path, error } => {
8887            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
8888        }
8889    }
8890}
8891
8892/// Assign a freshly spawned Windows child without ever resuming on failure.
8893///
8894/// Steps 2 and 3 of the suspended-create contract: the job is created and the
8895/// child assigned **while it is still suspended** (step 1 is
8896/// `suspend_on_create_async` at the spawn site). The caller resumes only after
8897/// the child's roster admission is durable. Failed containment kills and reaps
8898/// the suspended child instead of running an unprotected module.
8899#[cfg(windows)]
8900fn contain_spawned_child(
8901    child: &mut Child,
8902    spec: &ModuleSpec,
8903) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
8904    let assignment = || {
8905        let job = subc_jobobject::JobObject::new()?;
8906        #[cfg(test)]
8907        spawn_record_tests::assignment_failpoint(spec)?;
8908        job.assign(child)?;
8909        Ok::<_, io::Error>(job)
8910    };
8911    match assignment() {
8912        Ok(job) => Ok(Some(job)),
8913        Err(source) => {
8914            kill_and_reap_rejected_child(child);
8915            Err(SuperviseError::Spawn {
8916                program: spec.program.clone(),
8917                source: io::Error::other(format!(
8918                    "Windows module job containment failed: {source}"
8919                )),
8920                cgroup_path: None,
8921            })
8922        }
8923    }
8924}
8925
8926/// Rejection is synchronous: do not return a spawn error while the child can
8927/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
8928fn kill_and_reap_rejected_child(child: &mut Child) {
8929    #[cfg(unix)]
8930    if let Some(pid) = child
8931        .id()
8932        .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
8933    {
8934        let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
8935    }
8936    let _ = child.start_kill();
8937    while matches!(child.try_wait(), Ok(None)) {
8938        std::thread::yield_now();
8939    }
8940}
8941
8942/// Resume a suspended child, killing it if it cannot be started.
8943///
8944/// A suspended process holds a pid and does nothing, so there is no useful
8945/// state to return: the caller gets an error and the spawn fails.
8946#[cfg(windows)]
8947fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
8948    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
8949        // Kill it here rather than leaving a suspended process for the caller
8950        // to notice; `kill_on_drop` would eventually do this, but the module
8951        // would have been reported as running in between.
8952        let _ = std::process::Command::new("taskkill.exe")
8953            .args(["/PID", &pid.to_string(), "/T", "/F"])
8954            .stdin(Stdio::null())
8955            .stdout(Stdio::null())
8956            .stderr(Stdio::null())
8957            .status();
8958        return Err(SuperviseError::Spawn {
8959            program: spec.program.clone(),
8960            source,
8961            cgroup_path: None,
8962        });
8963    }
8964    Ok(())
8965}
8966
8967#[cfg(target_os = "linux")]
8968fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
8969    match placement.remove_module(module_id) {
8970        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
8971        Err(error) => warn!(
8972            module_id,
8973            error = %error,
8974            "could not remove module cgroup after process exit; continuing teardown"
8975        ),
8976    }
8977}
8978
8979#[cfg(target_os = "linux")]
8980async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
8981    // Reaping the direct child is not proof its descendants exited. End the
8982    // residual tree and wait for the kernel's population fact before rmdir;
8983    // otherwise a successful parent wait leaks a directory on each restart.
8984    if matches!(
8985        subc_cgroup::kill_module(Some(placement), module_id),
8986        subc_cgroup::KillOutcome::Killed
8987    ) {
8988        if let Ok(path) = placement.module_path(module_id) {
8989            while std::fs::read_to_string(path.join("cgroup.events"))
8990                .ok()
8991                .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8992            {
8993                sleep(Duration::from_millis(1)).await;
8994            }
8995        }
8996    }
8997    remove_module_cgroup(placement, module_id);
8998}
8999
9000#[cfg(target_os = "linux")]
9001fn apply_cgroup_placement(
9002    command: &mut Command,
9003    spec: &ModuleSpec,
9004    path: &std::path::Path,
9005) -> Result<(), SuperviseError> {
9006    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9007        module_id: spec.module_id.clone(),
9008        source,
9009    })
9010}
9011
9012fn capture_retention(spec: &ModuleSpec) -> Retention {
9013    let defaults = Retention::default();
9014    let value = |name: &str| {
9015        spec.env
9016            .iter()
9017            .rev()
9018            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9019    };
9020    Retention {
9021        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9022            .and_then(|value| value.parse().ok())
9023            .unwrap_or(defaults.max_file_mb),
9024        keep: value(CAPTURE_KEEP_ENV)
9025            .and_then(|value| value.parse().ok())
9026            .unwrap_or(defaults.keep),
9027        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9028            .and_then(|value| value.parse().ok())
9029            .unwrap_or(defaults.max_age_days),
9030    }
9031}
9032
9033/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
9034/// module's registration to the exact process the supervisor spawned.
9035fn generate_launch_nonce() -> Result<String, SuperviseError> {
9036    let mut bytes = [0u8; 32];
9037    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9038        reason: source.to_string(),
9039    })?;
9040    let mut hex = String::with_capacity(64);
9041    for b in bytes {
9042        use std::fmt::Write;
9043        let _ = write!(hex, "{b:02x}");
9044    }
9045    Ok(hex)
9046}
9047
9048/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
9049/// signal about how many leading bytes matched.
9050fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9051    if a.len() != b.len() {
9052        return false;
9053    }
9054    let mut diff = 0u8;
9055    for (x, y) in a.iter().zip(b.iter()) {
9056        diff |= x ^ y;
9057    }
9058    diff == 0
9059}
9060
9061/// The kernel's image after an acknowledged exec, shared by ordinary launches
9062/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
9063/// not identities inferred from a configured pathname.
9064#[cfg(not(windows))]
9065fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9066    let process = subc_os::Process::open(pid).ok().flatten()?;
9067    #[cfg(target_os = "linux")]
9068    return sample_after_linux_exec(
9069        || {
9070            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9071            let committed = linux_exec_committed(&stat);
9072            #[cfg(test)]
9073            spawn_image_tests::report_exec_state(committed);
9074            committed
9075        },
9076        || process.observe(),
9077    );
9078    #[cfg(not(target_os = "linux"))]
9079    process.observe()
9080}
9081
9082/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
9083/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
9084/// installation. Until it clears, /proc/pid/exe can still name the daemon.
9085#[cfg(target_os = "linux")]
9086fn linux_exec_committed(stat: &str) -> Option<bool> {
9087    const PF_FORKNOEXEC: u64 = 0x40;
9088    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9089    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9090}
9091
9092#[cfg(target_os = "linux")]
9093fn sample_after_linux_exec(
9094    mut exec_committed: impl FnMut() -> Option<bool>,
9095    observe: impl FnOnce() -> Option<subc_os::Observation>,
9096) -> Option<subc_os::Observation> {
9097    let deadline = std::time::Instant::now() + Duration::from_secs(1);
9098    while !exec_committed()? {
9099        // A failed or indefinitely stalled exec must not publish the inherited
9100        // image as authority to signal a future orphan. Missing identity keeps
9101        // the existing conservative orphan-sweep behavior.
9102        if std::time::Instant::now() >= deadline {
9103            return None;
9104        }
9105        std::thread::sleep(Duration::from_micros(50));
9106    }
9107    observe()
9108}
9109
9110#[cfg(all(test, target_os = "linux"))]
9111mod spawn_image_tests {
9112    use super::*;
9113    use std::cell::{Cell, RefCell};
9114
9115    enum ExecEvent {
9116        Uncommitted,
9117        Image(Option<subc_os::Observation>),
9118    }
9119
9120    thread_local! {
9121        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9122    }
9123
9124    pub(super) fn report_exec_state(committed: Option<bool>) {
9125        if committed == Some(false) {
9126            EXEC_EVENTS.with(|events| {
9127                if let Some(events) = events.borrow_mut().take() {
9128                    let _ = events.send(ExecEvent::Uncommitted);
9129                }
9130            });
9131        }
9132    }
9133
9134    #[test]
9135    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9136        use std::io::{Read, Write};
9137        use std::os::fd::AsRawFd;
9138
9139        let descriptor = std::fs::File::open("/dev/null").unwrap();
9140        let mut command = std::process::Command::new("/bin/sleep");
9141        command.arg("30");
9142        let (mut ready, mut release) =
9143            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9144                .unwrap();
9145        let spawn = std::thread::spawn(move || command.spawn());
9146        let mut message = [0u8; 8];
9147        ready.read_exact(&mut message).unwrap();
9148        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9149        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9150        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9151        let (events, received) = std::sync::mpsc::channel();
9152        let observer = std::thread::spawn(move || {
9153            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9154            let image = observe_spawned_image(pid);
9155            let _ = events.send(ExecEvent::Image(image));
9156        });
9157        // Release only once the production observer has seen the uncommitted
9158        // exec, or has incorrectly returned the inherited image. Either way,
9159        // release and reap before asserting so a regression cannot leak a child.
9160        let first = received.recv_timeout(Duration::from_secs(5));
9161        release.write_all(&[1]).unwrap();
9162        let mut child = spawn.join().unwrap().unwrap();
9163        let after = match first {
9164            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9165                Ok(ExecEvent::Image(image)) => image,
9166                _ => None,
9167            },
9168            Ok(ExecEvent::Image(image)) => image,
9169            Err(_) => None,
9170        };
9171        observer.join().unwrap();
9172        child.kill().unwrap();
9173        child.wait().unwrap();
9174
9175        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9176        assert_eq!(
9177            inherited.unwrap().executable,
9178            subc_os::file_identity(&std::env::current_exe().unwrap())
9179        );
9180        assert_eq!(
9181            after.unwrap().executable,
9182            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9183        );
9184    }
9185
9186    #[test]
9187    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9188        let committed = Cell::new(false);
9189        let inherited = subc_os::Observation {
9190            start_time: 23,
9191            executable: Some(subc_os::FileIdentity {
9192                device: 29,
9193                inode: 31,
9194            }),
9195        };
9196        let module = subc_os::Observation {
9197            start_time: 23,
9198            executable: Some(subc_os::FileIdentity {
9199                device: 29,
9200                inode: 37,
9201            }),
9202        };
9203        let polls = Cell::new(0);
9204        let image = sample_after_linux_exec(
9205            || {
9206                polls.set(polls.get() + 1);
9207                // Hold the inherited image through the first poll, as in the
9208                // posix_spawn parent-wake window, then commit the module image.
9209                committed.set(polls.get() == 2);
9210                Some(committed.get())
9211            },
9212            || Some(if committed.get() { module } else { inherited }),
9213        );
9214        assert_eq!(image, Some(module));
9215        assert_eq!(polls.get(), 2);
9216    }
9217
9218    #[test]
9219    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9220        assert_eq!(
9221            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9222            Some(false)
9223        );
9224        assert_eq!(
9225            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9226            Some(true)
9227        );
9228        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9229        assert_eq!(linux_exec_committed("17 (a) R"), None);
9230    }
9231
9232    #[test]
9233    fn unreadable_exec_state_never_samples_an_inherited_image() {
9234        assert_eq!(
9235            sample_after_linux_exec(
9236                || None,
9237                || panic!("must not sample before exec confirmation")
9238            ),
9239            None
9240        );
9241    }
9242
9243    #[test]
9244    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9245        assert_eq!(
9246            sample_after_linux_exec(
9247                || Some(false),
9248                || panic!("must not sample after an exec timeout")
9249            ),
9250            None
9251        );
9252    }
9253}
9254
9255/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
9256/// different executable after the no-delete capture was taken.
9257#[cfg(windows)]
9258fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9259    let mut program = spec.program.clone();
9260    if program.is_absolute() || program.components().count() > 1 {
9261        // An explicit path can name a PE image with no extension. Try that
9262        // exact file first, as Command does, before falling back to .exe.
9263        match subc_os::ExecutableCapture::open(&program) {
9264            Ok(capture) => return Ok(capture),
9265            Err(error)
9266                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9267            Err(error) => return Err(error),
9268        }
9269        program.set_extension("exe");
9270        return subc_os::ExecutableCapture::open(&program);
9271    }
9272    if program.extension().is_none() {
9273        program.set_extension("exe");
9274    }
9275    let custom_path = spec
9276        .env
9277        .iter()
9278        .rev()
9279        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9280    let mut directories = Vec::new();
9281    // These are Windows' default executable search locations. A configured
9282    // PATH still overrides the inherited PATH, not the system directories.
9283    if let Some(parent) = std::env::current_exe()?.parent() {
9284        directories.push(parent.to_path_buf());
9285    }
9286    directories.push(std::env::current_dir()?);
9287    if let Some(windows) = std::env::var_os("SystemRoot") {
9288        let windows = PathBuf::from(windows);
9289        directories.push(windows.join("System32"));
9290        directories.push(windows.clone());
9291    }
9292    let path = custom_path
9293        .map(|(_, value)| std::ffi::OsString::from(value))
9294        .or_else(|| std::env::var_os("PATH"));
9295    if let Some(path) = path {
9296        directories.extend(std::env::split_paths(&path));
9297    }
9298    for directory in directories {
9299        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9300            Ok(capture) => return Ok(capture),
9301            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9302            Err(error) => return Err(error),
9303        }
9304    }
9305    Err(io::Error::new(
9306        io::ErrorKind::NotFound,
9307        "resolved executable not found",
9308    ))
9309}
9310
9311fn spawn_and_mark_running(
9312    spec: &ModuleSpec,
9313    runtime: &SupervisorRuntimeConfig,
9314    snapshot: &SharedSnapshot,
9315) -> Result<SupervisedChild, SuperviseError> {
9316    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9317    let child = spawn_child(
9318        spec,
9319        runtime.connection_file_path.as_deref(),
9320        runtime.supervisor_handle.as_ref(),
9321        &runtime.stderr_ring,
9322        runtime.capture_logs_dir.as_deref(),
9323        &runtime.child_roster,
9324        #[cfg(target_os = "linux")]
9325        runtime.cgroup_placement.as_ref(),
9326    )?;
9327    set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9328    if let Some(spawn) = activity_spawn {
9329        spawn.complete(snapshot);
9330    }
9331    Ok(child)
9332}
9333
9334enum RegistrationWaitOutcome {
9335    Registered,
9336    Exited(ExitReport),
9337    TimedOut,
9338}
9339
9340struct ReloadRegistrationFailure {
9341    exit_report: ExitReport,
9342    reason: String,
9343}
9344
9345#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9346enum BusyGaugeObservation {
9347    Quiescent,
9348    Busy,
9349    Omitted,
9350}
9351
9352fn busy_gauge_observation(metrics: Option<&Value>, gauges: &[String]) -> BusyGaugeObservation {
9353    let Some(metrics) = metrics.and_then(Value::as_object) else {
9354        return BusyGaugeObservation::Omitted;
9355    };
9356    let mut sum = 0u128;
9357    for gauge in gauges {
9358        let Some(value) = metrics.get(gauge) else {
9359            return BusyGaugeObservation::Omitted;
9360        };
9361        let Some(value) = value.as_u64() else {
9362            return BusyGaugeObservation::Busy;
9363        };
9364        sum = sum.saturating_add(u128::from(value));
9365    }
9366    if sum == 0 {
9367        BusyGaugeObservation::Quiescent
9368    } else {
9369        BusyGaugeObservation::Busy
9370    }
9371}
9372
9373fn declared_busy_gauges(
9374    registry: &Registry,
9375    module_id: &str,
9376) -> Result<Vec<String>, SuperviseError> {
9377    busy_gauges_of(
9378        registry
9379            .get_module(module_id)
9380            .map_err(SuperviseError::Registry)?,
9381    )
9382}
9383
9384/// [`declared_busy_gauges`] for the registration a connection holds, in any
9385/// slot: after cutover the incumbent is no longer the id's active
9386/// registration, and its own manifest is the one that names its gauges.
9387fn declared_busy_gauges_for_connection(
9388    registry: &Registry,
9389    connection_id: ConnectionId,
9390) -> Result<Vec<String>, SuperviseError> {
9391    busy_gauges_of(
9392        registry
9393            .get_module_by_connection(connection_id)
9394            .map_err(SuperviseError::Registry)?,
9395    )
9396}
9397
9398fn busy_gauges_of(
9399    registration: Option<crate::registry::ModuleRegistration>,
9400) -> Result<Vec<String>, SuperviseError> {
9401    let Some(registration) = registration else {
9402        return Ok(Vec::new());
9403    };
9404    let Some(self_signals) = registration.manifest.self_signals else {
9405        return Ok(Vec::new());
9406    };
9407
9408    let mut gauges = Vec::new();
9409    for declaration in self_signals {
9410        if declaration.kind != SelfSignalKind::Busy {
9411            continue;
9412        }
9413        match declaration.anchored_to {
9414            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9415                gauges.extend(declared)
9416            }
9417            _ => {
9418                // An invalid Busy anchor is fail-safe: the empty name cannot be
9419                // present in a conforming health report, so this drain stays busy.
9420                gauges.push(String::new());
9421            }
9422        }
9423    }
9424    Ok(gauges)
9425}
9426
9427/// Wait for `endpoint` to have nothing in flight and, when the module declares
9428/// busy gauges, for a health probe to report them quiet. The probe is addressed
9429/// by `scope`: a swap's superseded incumbent must be asked about its own
9430/// gauges, and by module id the probe would reach the promoted candidate.
9431/// While the gauges read busy, the probe repeats every
9432/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
9433async fn wait_for_forwarding_quiescence(
9434    forwarding: &ForwardingTable,
9435    module_id: &str,
9436    runtime: &SupervisorRuntimeConfig,
9437    endpoint: crate::ModuleEndpointId,
9438    deadline: Instant,
9439    busy_gauges: &[String],
9440    scope: DrainScope,
9441) -> Result<bool, SuperviseError> {
9442    let mut gauges_quiescent = busy_gauges.is_empty();
9443    let mut next_probe_at = Instant::now();
9444    let mut omission_counted = false;
9445
9446    loop {
9447        let now = Instant::now();
9448        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9449            let report = match scope {
9450                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9451                DrainScope::Endpoint(endpoint) => {
9452                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9453                }
9454            };
9455            gauges_quiescent = match report {
9456                Ok(report) => match busy_gauge_observation(report.metrics.as_ref(), busy_gauges) {
9457                    BusyGaugeObservation::Quiescent => true,
9458                    BusyGaugeObservation::Busy => false,
9459                    BusyGaugeObservation::Omitted => {
9460                        if !omission_counted {
9461                            forwarding
9462                                .counters()
9463                                .increment_drains_with_undeclared_gauge();
9464                            omission_counted = true;
9465                        }
9466                        false
9467                    }
9468                },
9469                Err(err) => {
9470                    warn!(
9471                        module_id,
9472                        error = %err,
9473                        "drain health.check did not produce declared busy gauges; treating module as busy"
9474                    );
9475                    false
9476                }
9477            };
9478            // Counted from the probe's completion: the probe above was awaited,
9479            // so no second probe can start while one is still outstanding.
9480            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
9481        }
9482
9483        let in_flight = forwarding
9484            .endpoint_in_flight_count(endpoint)
9485            .map_err(SuperviseError::Forwarding)?;
9486        if in_flight == 0 && gauges_quiescent {
9487            return Ok(true);
9488        }
9489
9490        let now = Instant::now();
9491        if now >= deadline {
9492            return Ok(false);
9493        }
9494        let mut wait = deadline
9495            .saturating_duration_since(now)
9496            .min(REGISTRY_RELEASE_POLL);
9497        if !busy_gauges.is_empty() {
9498            wait = wait.min(next_probe_at.saturating_duration_since(now));
9499        }
9500        sleep(wait).await;
9501    }
9502}
9503
9504/// The wait between drain busy-gauge probes: the larger of
9505/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
9506/// drain's own in-flight poll interval.
9507fn drain_busy_reprobe_interval() -> Duration {
9508    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
9509}
9510
9511/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
9512///
9513/// `Ok` is always honest and passed straight through -- the wait actually measured
9514/// in-flight state. `Err` means the wait produced no measurement at all (the
9515/// forwarding table's lock was poisoned), so `false` is reported as the one honest
9516/// constant: the drain did not complete. Never recomputed from route state, never a
9517/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
9518fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
9519    match wait_result {
9520        Ok(drained) => *drained,
9521        Err(_) => false,
9522    }
9523}
9524
9525fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
9526    for released in released_routes {
9527        let frame = match Frame::build_with_version(
9528            released.negotiated_ver,
9529            FrameType::Goodbye,
9530            control_flags(),
9531            released.channel,
9532            released.epoch,
9533            0,
9534            Vec::new(),
9535        ) {
9536            Ok(frame) => frame,
9537            Err(err) => {
9538                warn!(
9539                    route_channel = released.channel,
9540                    error = %err,
9541                    "failed to build supervisor drain route GOODBYE frame"
9542                );
9543                continue;
9544            }
9545        };
9546        if !released.close_on_delivery_failure() {
9547            crate::forwarding::send_module_route_goodbye(
9548                &forwarding.counters(),
9549                &released.sink,
9550                frame,
9551                released.module_id.as_deref(),
9552                "supervisor drain",
9553            );
9554            continue;
9555        }
9556        if let Err(err) = released.sink.try_send(frame) {
9557            warn!(
9558                target_connection_id = released.connection_id.get(),
9559                route_channel = released.channel,
9560                error = %err,
9561                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
9562            );
9563            let _ = forwarding.escalate_client_delivery_failure(
9564                released.connection_id,
9565                released.channel,
9566                released.epoch,
9567                CloseReason::new(
9568                    "route_goodbye_delivery_failed",
9569                    format!(
9570                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
9571                        released.channel
9572                    ),
9573                ),
9574                crate::forwarding::UndeliveredFrame {
9575                    module_id: released.module_id.as_deref(),
9576                    sink: &released.sink,
9577                },
9578            );
9579        }
9580    }
9581}
9582
9583fn send_module_draining(
9584    module_id: &str,
9585    reason: RouteCloseReason,
9586    deadline_ms: u64,
9587    target: &ModuleDrainTarget,
9588) {
9589    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
9590        reason,
9591        deadline_ms,
9592    }) {
9593        Ok(body) => body,
9594        Err(err) => {
9595            warn!(
9596                module_id,
9597                error = %err,
9598                "failed to encode module draining command"
9599            );
9600            return;
9601        }
9602    };
9603    let frame = match Frame::build_with_version(
9604        target.negotiated_ver,
9605        FrameType::Push,
9606        control_flags(),
9607        0,
9608        0,
9609        0,
9610        body,
9611    ) {
9612        Ok(frame) => frame,
9613        Err(err) => {
9614            warn!(
9615                module_id,
9616                error = %err,
9617                "failed to build module draining command frame"
9618            );
9619            return;
9620        }
9621    };
9622    if let Err(err) = target.sink.try_send(frame) {
9623        warn!(
9624            module_id,
9625            target_connection_id = target.endpoint.connection_id.get(),
9626            error = %err,
9627            "module draining command was not delivered to peer"
9628        );
9629    }
9630}
9631
9632/// The channel-0 GOODBYE that tells a module its stop is planned.
9633fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
9634    match Frame::build_with_version(
9635        negotiated_ver,
9636        FrameType::Goodbye,
9637        control_flags(),
9638        0,
9639        0,
9640        0,
9641        Vec::new(),
9642    ) {
9643        Ok(frame) => Some(frame),
9644        Err(err) => {
9645            warn!(
9646                module_id,
9647                error = %err,
9648                "failed to build module GOODBYE frame"
9649            );
9650            None
9651        }
9652    }
9653}
9654
9655/// Send every registered module connection its module GOODBYE at daemon
9656/// shutdown, then request that connection's close.
9657///
9658/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
9659/// before EOF, so the GOODBYE must reach the socket before the close. A close
9660/// request does not wait for the connection's queued frames: its writer gets a
9661/// bounded grace after the close, is aborted if it overruns it, and the daemon
9662/// process may exit before that grace ends. So with `wait_for_flush`, each
9663/// connection is closed only after its writer has acknowledged writing the
9664/// GOODBYE, or once a short shared budget runs out, so one module that is not
9665/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
9666/// are only queued, for a shutdown the operator has told to stop waiting.
9667/// A connection that is already gone is skipped.
9668#[cfg(unix)]
9669async fn send_module_goodbyes_for_daemon_shutdown(
9670    forwarding: &Arc<ForwardingTable>,
9671    reason: &CloseReason,
9672    wait_for_flush: bool,
9673) {
9674    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
9675    let targets = match forwarding.module_connections() {
9676        Ok(targets) => targets,
9677        Err(err) => {
9678            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
9679            return;
9680        }
9681    };
9682    let deadline = Instant::now() + GOODBYE_BUDGET;
9683    let mut sends = tokio::task::JoinSet::new();
9684    for target in targets {
9685        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
9686            continue;
9687        };
9688        if !wait_for_flush {
9689            if let Err(err) = target.sink.try_send(frame) {
9690                debug!(
9691                    module_id = %target.module_id,
9692                    error = %err,
9693                    "shutdown module GOODBYE was not queued"
9694                );
9695            }
9696            continue;
9697        }
9698        let forwarding = Arc::clone(forwarding);
9699        let reason = reason.clone();
9700        sends.spawn(async move {
9701            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
9702                Ok(Ok(())) => {}
9703                Ok(Err(err)) => debug!(
9704                    module_id = %target.module_id,
9705                    error = %err,
9706                    "module connection closed before its shutdown GOODBYE was written"
9707                ),
9708                Err(_) => warn!(
9709                    module_id = %target.module_id,
9710                    budget = ?GOODBYE_BUDGET,
9711                    "shutdown module GOODBYE was not written within its budget; closing anyway"
9712                ),
9713            }
9714            forwarding.request_connection_close(target.endpoint.connection_id, reason);
9715        });
9716    }
9717    // Every task ends by the shared deadline, so this wait is bounded too.
9718    while sends.join_next().await.is_some() {}
9719}
9720
9721fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
9722    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
9723        return;
9724    };
9725    if let Err(err) = target.sink.try_send(frame) {
9726        warn!(
9727            module_id,
9728            target_connection_id = target.endpoint.connection_id.get(),
9729            error = %err,
9730            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
9731        );
9732        forwarding.request_connection_close(
9733            target.endpoint.connection_id,
9734            CloseReason::new(
9735                "module_goodbye_delivery_failed",
9736                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
9737            ),
9738        );
9739    }
9740}
9741
9742#[derive(Clone, Copy)]
9743struct ForwardingDrainContext<'a> {
9744    spec: &'a ModuleSpec,
9745    runtime: &'a SupervisorRuntimeConfig,
9746    registry: &'a Registry,
9747    scope: DrainScope,
9748}
9749
9750/// Which process a forwarding drain addresses.
9751#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9752enum DrainScope {
9753    /// Whatever endpoint is active for the module id: every plain stop,
9754    /// restart and reload. Also moves the module's state to `Draining`.
9755    Active,
9756    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
9757    /// module id would resolve to the promoted candidate and leave neither
9758    /// process routable. The module's state is left alone, since the promoted
9759    /// candidate is what it describes and that process is running.
9760    Endpoint(crate::ModuleEndpointId),
9761}
9762
9763/// Whether a child being drained has already been asked to stop by the time
9764/// its drain wait starts.
9765///
9766/// The drain wait is the same budget whatever this says. What it decides is
9767/// whether the supervisor must ask by signal before that wait begins: a child
9768/// that nobody asked will sit out the whole budget and then be SIGKILLed,
9769/// healthy or not.
9770#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9771enum StopNotice {
9772    /// The module was sent `module.draining` and a module GOODBYE over its own
9773    /// registered connection, and stops itself.
9774    SentOverConnection,
9775    /// The forwarding drain found no registered connection for the module: a
9776    /// subc child spawned moments ago that has not sent HELLO yet, or a
9777    /// `protocol: "none"` child, which never registers.
9778    NoConnection,
9779    /// This path sends nothing over the module's connection: the supervisor has
9780    /// no forwarding table, or the caller stops the child without a forwarding
9781    /// drain.
9782    NotSent,
9783}
9784
9785async fn begin_forwarding_drain(
9786    spec: &ModuleSpec,
9787    runtime: &SupervisorRuntimeConfig,
9788    registry: &Registry,
9789    snapshot: &SharedSnapshot,
9790    enabled: Option<bool>,
9791    reason: RouteCloseReason,
9792) -> Result<StopNotice, SuperviseError> {
9793    let Some(forwarding) = runtime.forwarding.as_ref() else {
9794        return Err(SuperviseError::ReloadUnavailable {
9795            module_id: spec.module_id.clone(),
9796            reason: "supervisor was not configured with a forwarding table".to_string(),
9797        });
9798    };
9799
9800    begin_forwarding_drain_with(
9801        forwarding,
9802        ForwardingDrainContext {
9803            spec,
9804            runtime,
9805            registry,
9806            scope: DrainScope::Active,
9807        },
9808        snapshot,
9809        enabled,
9810        reason,
9811        runtime.drain_timeout,
9812    )
9813    .await
9814}
9815
9816async fn begin_forwarding_drain_if_configured(
9817    spec: &ModuleSpec,
9818    runtime: &SupervisorRuntimeConfig,
9819    registry: &Registry,
9820    snapshot: &SharedSnapshot,
9821    enabled: Option<bool>,
9822    reason: RouteCloseReason,
9823) -> Result<StopNotice, SuperviseError> {
9824    begin_forwarding_drain_with_timeout(
9825        spec,
9826        runtime,
9827        registry,
9828        snapshot,
9829        enabled,
9830        reason,
9831        runtime.drain_timeout,
9832    )
9833    .await
9834}
9835
9836/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
9837/// budget, for paths where the operator overrides the module's configured one
9838/// (`supervisor.restart{drain_timeout_ms}`).
9839async fn begin_forwarding_drain_with_timeout(
9840    spec: &ModuleSpec,
9841    runtime: &SupervisorRuntimeConfig,
9842    registry: &Registry,
9843    snapshot: &SharedSnapshot,
9844    enabled: Option<bool>,
9845    reason: RouteCloseReason,
9846    drain_timeout: Duration,
9847) -> Result<StopNotice, SuperviseError> {
9848    let Some(forwarding) = runtime.forwarding.as_ref() else {
9849        return Ok(StopNotice::NotSent);
9850    };
9851
9852    begin_forwarding_drain_with(
9853        forwarding,
9854        ForwardingDrainContext {
9855            spec,
9856            runtime,
9857            registry,
9858            scope: DrainScope::Active,
9859        },
9860        snapshot,
9861        enabled,
9862        reason,
9863        drain_timeout,
9864    )
9865    .await
9866}
9867
9868async fn begin_forwarding_drain_with(
9869    forwarding: &ForwardingTable,
9870    context: ForwardingDrainContext<'_>,
9871    snapshot: &SharedSnapshot,
9872    enabled: Option<bool>,
9873    reason: RouteCloseReason,
9874    drain_timeout: Duration,
9875) -> Result<StopNotice, SuperviseError> {
9876    let ForwardingDrainContext {
9877        spec,
9878        runtime,
9879        registry,
9880        scope,
9881    } = context;
9882    debug_assert_ne!(reason, RouteCloseReason::Crash);
9883    let terminal = matches!(reason, RouteCloseReason::Disable);
9884    let drain_started_at = Instant::now();
9885    let drain_deadline = drain_started_at + drain_timeout;
9886    let deadline_ms =
9887        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
9888    let busy_gauges = match scope {
9889        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
9890        DrainScope::Endpoint(endpoint) => {
9891            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
9892        }
9893    };
9894
9895    // Admission gate first: route.open/commit and route REQUEST admission are closed
9896    // before the first quiescence check, so the outstanding count can only fall.
9897    let gate_started = Instant::now();
9898    let drain_target = match scope {
9899        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
9900        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
9901    }
9902    .map_err(SuperviseError::Forwarding)?;
9903    // The instant admission closed, and how long taking the forwarding write
9904    // lock to close it took. The timeout line reports only the quiescence
9905    // wait, so without this a drain that started late looked like one that
9906    // started on time.
9907    info!(
9908        module_id = %spec.module_id,
9909        ?reason,
9910        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
9911        connected = drain_target.is_some(),
9912        "module drain began; route admission closed"
9913    );
9914    if scope == DrainScope::Active {
9915        update_snapshot(snapshot, Some(&spec.module_id), |state| {
9916            state.state = ModuleState::Draining;
9917            state.draining_to_replace =
9918                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
9919            if let Some(enabled) = enabled {
9920                state.enabled = enabled;
9921            }
9922        })?;
9923    }
9924
9925    let Some(target) = drain_target.as_ref() else {
9926        // Nothing was sent: the module has no registered connection to carry
9927        // `module.draining` or a GOODBYE. The caller must not assume the child
9928        // was asked to stop.
9929        return Ok(StopNotice::NoConnection);
9930    };
9931    {
9932        send_module_draining(&spec.module_id, reason, deadline_ms, target);
9933        #[cfg(feature = "test-support")]
9934        forwarding.wait_before_route_closing_for_test().await;
9935        let routes = forwarding
9936            .endpoint_routes(target.endpoint)
9937            .map_err(SuperviseError::Forwarding)?;
9938        let routes_notified = routes.len();
9939        crate::control::send_route_control_pushes(
9940            forwarding,
9941            routes.clone(),
9942            ClientControlPush::RouteClosing {
9943                module_id: spec.module_id.clone(),
9944                channels: Vec::new(),
9945                reason,
9946            },
9947        );
9948        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
9949
9950        // `route.closing` was just sent above: from here on every return path,
9951        // including an early one, MUST send `route.closed` before propagating
9952        // anything else. A client holds `closing` as a promise that a verdict is
9953        // coming; leaving early without `closed` strands it waiting forever, since
9954        // `closing` carries no timeout of its own.
9955        let wait_result = wait_for_forwarding_quiescence(
9956            forwarding,
9957            &spec.module_id,
9958            runtime,
9959            target.endpoint,
9960            drain_deadline,
9961            &busy_gauges,
9962            scope,
9963        )
9964        .await;
9965        let drained = drained_after_quiescence_wait(&wait_result);
9966        if let Err(err) = &wait_result {
9967            error!(
9968                module_id = %spec.module_id,
9969                ?reason,
9970                error = %err,
9971                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
9972            );
9973        } else if !drained {
9974            // Name what the drain waited on. Without it the line says only that
9975            // something did not settle, and "one wedged call" and "every
9976            // session's held stream" read the same; the first is a module bug,
9977            // the second is a module that should end its streams on
9978            // module.draining. Read before teardown releases the routes.
9979            let holdouts = forwarding
9980                .endpoint_drain_holdouts(target.endpoint)
9981                .unwrap_or_default();
9982            warn!(
9983                module_id = %spec.module_id,
9984                waited = ?drain_timeout,
9985                ?reason,
9986                held_requests = holdouts.requests,
9987                held_routes = holdouts.routes,
9988                total_routes = holdouts.total_routes,
9989                top_connections = ?holdouts.top_connections,
9990                // `module_channel:corr`, so the module can find each held request
9991                // in its own log; capped, so `held_requests` is the full count.
9992                held = %holdouts
9993                    .held
9994                    .iter()
9995                    .map(|(channel, corr)| format!("{channel}:{corr}"))
9996                    .collect::<Vec<_>>()
9997                    .join(","),
9998                "route drain timed out before request quiescence; forcing teardown"
9999            );
10000        }
10001        crate::control::send_route_control_pushes(
10002            forwarding,
10003            routes,
10004            ClientControlPush::RouteClosed {
10005                module_id: spec.module_id.clone(),
10006                channels: Vec::new(),
10007                reason,
10008                drained,
10009                abandoned: target.abandoned_bindings.len() as u32,
10010                excluded_subscriptions: target.excluded_subscriptions,
10011                terminal: Some(terminal),
10012            },
10013        );
10014        wait_result?;
10015
10016        // `route.closed` has now been sent unconditionally above. From here the
10017        // remaining steps are cleanup (route + module GOODBYE) rather than a
10018        // promise the client is waiting on, but a lock-poisoned
10019        // `release_module_endpoint_routes` would otherwise skip the module
10020        // GOODBYE silently too -- send it before propagating the error.
10021        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10022            Ok(routes) => routes,
10023            Err(err) => {
10024                warn!(
10025                    module_id = %spec.module_id,
10026                    ?reason,
10027                    error = %err,
10028                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10029                );
10030                send_module_goodbye(&spec.module_id, forwarding, target);
10031                return Err(SuperviseError::Forwarding(err));
10032            }
10033        };
10034        let route_goodbye_count = released_routes.len();
10035        send_route_goodbyes(forwarding, released_routes);
10036        send_module_goodbye(&spec.module_id, forwarding, target);
10037
10038        // The drain's happy path was previously silent: every emission above is
10039        // best-effort with only its failure arm logged, so "were consumers told"
10040        // was unprovable from the daemon log (surfaced by a 30-minute consumer
10041        // hang where the open question was exactly whether teardown notice went
10042        // out). One summary line makes that class decidable in one grep.
10043        info!(
10044            module_id = %spec.module_id,
10045            ?reason,
10046            routes_notified,
10047            route_goodbyes = route_goodbye_count,
10048            abandoned_reservations = target.abandoned_bindings.len(),
10049            excluded_subscriptions = target.excluded_subscriptions,
10050            drained,
10051            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10052        );
10053    }
10054
10055    Ok(StopNotice::SentOverConnection)
10056}
10057
10058/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
10059/// the only slot a plain (non-swap) spawn can register into.
10060async fn wait_for_registration_after_reload(
10061    registry: &Registry,
10062    module_id: &str,
10063    snapshot: &SharedSnapshot,
10064    child: &mut SupervisedChild,
10065    wait: Duration,
10066) -> Result<RegistrationWaitOutcome, SuperviseError> {
10067    wait_for_slot_registration(
10068        registry,
10069        crate::registry::RegistrationSlot::Active(module_id),
10070        module_id,
10071        snapshot,
10072        child,
10073        wait,
10074    )
10075    .await
10076}
10077
10078/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
10079///
10080/// Keyed on the slot rather than the bare module id because during a swap the
10081/// id's active slot is already held by the incumbent: an id-keyed wait would
10082/// report the incumbent's registration as the candidate's and a candidate that
10083/// never registers would look registered. A swap candidate waits on
10084/// `crate::registry::RegistrationSlot::Candidate`.
10085async fn wait_for_slot_registration(
10086    registry: &Registry,
10087    slot: crate::registry::RegistrationSlot<'_>,
10088    module_id: &str,
10089    snapshot: &SharedSnapshot,
10090    child: &mut SupervisedChild,
10091    wait: Duration,
10092) -> Result<RegistrationWaitOutcome, SuperviseError> {
10093    let deadline = Instant::now() + wait;
10094    loop {
10095        if registry
10096            .registration(slot)
10097            .map_err(SuperviseError::Registry)?
10098            .is_some()
10099        {
10100            return Ok(RegistrationWaitOutcome::Registered);
10101        }
10102
10103        let now = Instant::now();
10104        if now >= deadline {
10105            return Ok(RegistrationWaitOutcome::TimedOut);
10106        }
10107        let remaining = deadline.saturating_duration_since(now);
10108        let poll = remaining.min(REGISTRY_RELEASE_POLL);
10109
10110        tokio::select! {
10111            wait_result = child.wait() => {
10112                let status = wait_result.map_err(|source| SuperviseError::Wait {
10113                    module_id: module_id.to_string(),
10114                    source,
10115                })?;
10116                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10117                    snapshot,
10118                    child,
10119                    &status,
10120                )));
10121            }
10122            _ = sleep(poll) => {}
10123        }
10124    }
10125}
10126
10127fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10128    // A replacement process that exits before HELLO did not provide service, even
10129    // if it used status 0. Count it against the restart cap as a new-binary failure.
10130    if exit_report.kind != ExitKind::DeliberateSeverance {
10131        exit_report.kind = ExitKind::Crash;
10132    }
10133    exit_report
10134}
10135
10136async fn handle_reload_child_registration_failure(
10137    spec: &ModuleSpec,
10138    runtime: &SupervisorRuntimeConfig,
10139    registry: &Registry,
10140    process_liveness: &SupervisorProcessLiveness,
10141    snapshot: &SharedSnapshot,
10142    _child: &mut Option<SupervisedChild>,
10143    failure: ReloadRegistrationFailure,
10144) -> Result<(), SuperviseError> {
10145    let ReloadRegistrationFailure {
10146        exit_report,
10147        reason,
10148    } = failure;
10149    match on_child_exit(
10150        spec,
10151        runtime.restart_policy,
10152        registry,
10153        snapshot,
10154        &runtime.terminal_ring,
10155        &runtime.spawn_events,
10156        &runtime.child_roster,
10157        exit_report,
10158    )
10159    .await
10160    {
10161        NextAction::Stop {
10162            registration_released,
10163        } => {
10164            if registration_released {
10165                process_liveness.untrack_if_current(&spec.module_id, snapshot);
10166            }
10167        }
10168        NextAction::Restart { schedule } => {
10169            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10170                schedule.delay
10171            });
10172            if let Some(schedule) = schedule {
10173                log_crash_respawn(&spec.module_id, schedule);
10174            }
10175            schedule_respawn(
10176                runtime,
10177                snapshot,
10178                &spec.module_id,
10179                delay,
10180                RespawnKind::Spawn,
10181            )?;
10182        }
10183    }
10184    Err(SuperviseError::ReloadFailed {
10185        module_id: spec.module_id.clone(),
10186        reason,
10187    })
10188}
10189
10190async fn handle_reload_spawn_failure(
10191    spec: &ModuleSpec,
10192    runtime: &SupervisorRuntimeConfig,
10193    process_liveness: &SupervisorProcessLiveness,
10194    snapshot: &SharedSnapshot,
10195    _child: &mut Option<SupervisedChild>,
10196    reason: String,
10197) -> Result<(), SuperviseError> {
10198    let now = Instant::now();
10199    let mut schedule = None;
10200    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10201        clear_current_process_facts(state);
10202        if state.enabled {
10203            schedule = state.next_crash_restart(&runtime.restart_policy, now);
10204            state.state = if schedule.is_some() {
10205                ModuleState::Restarting
10206            } else {
10207                ModuleState::Failed
10208            };
10209        } else {
10210            state.state = ModuleState::Disabled;
10211        }
10212    })?;
10213    if let Some(schedule) = schedule {
10214        schedule_respawn(
10215            runtime,
10216            snapshot,
10217            &spec.module_id,
10218            schedule.delay,
10219            RespawnKind::Spawn,
10220        )?;
10221    } else {
10222        process_liveness.untrack_if_current(&spec.module_id, snapshot);
10223    }
10224    Err(SuperviseError::ReloadFailed {
10225        module_id: spec.module_id.clone(),
10226        reason,
10227    })
10228}
10229
10230fn control_flags() -> Flags {
10231    Flags::new(false, Priority::Passive, false)
10232}
10233
10234#[allow(clippy::too_many_arguments)]
10235async fn drain_optional_child(
10236    module_id: &str,
10237    protocol: ModuleProtocol,
10238    stop_notice: StopNotice,
10239    registry: &Registry,
10240    forwarding: Option<&ForwardingTable>,
10241    snapshot: &SharedSnapshot,
10242    terminal_ring: &Arc<Mutex<TerminalRing>>,
10243    spawn_events: &SpawnEventFeed,
10244    child: &mut Option<SupervisedChild>,
10245    drain_timeout: Duration,
10246    final_state: ModuleState,
10247    enabled: Option<bool>,
10248) -> Result<(), SuperviseError> {
10249    if let Some(child) = child.take() {
10250        drain_child_to_state(
10251            module_id,
10252            protocol,
10253            stop_notice,
10254            registry,
10255            forwarding,
10256            snapshot,
10257            terminal_ring,
10258            spawn_events,
10259            child,
10260            drain_timeout,
10261            final_state,
10262            enabled,
10263        )
10264        .await
10265    } else {
10266        update_snapshot(snapshot, Some(module_id), |state| {
10267            state.state = final_state;
10268            if let Some(enabled) = enabled {
10269                state.enabled = enabled;
10270            }
10271            clear_current_process_facts(state);
10272        })?;
10273        release_dead_registration(registry, forwarding, snapshot, module_id).await
10274    }
10275}
10276
10277#[allow(clippy::too_many_arguments)]
10278async fn drain_child_to_state(
10279    module_id: &str,
10280    _protocol: ModuleProtocol,
10281    stop_notice: StopNotice,
10282    registry: &Registry,
10283    forwarding: Option<&ForwardingTable>,
10284    snapshot: &SharedSnapshot,
10285    terminal_ring: &Arc<Mutex<TerminalRing>>,
10286    spawn_events: &SpawnEventFeed,
10287    mut child: SupervisedChild,
10288    drain_timeout: Duration,
10289    final_state: ModuleState,
10290    enabled: Option<bool>,
10291) -> Result<(), SuperviseError> {
10292    let protocol = child.protocol;
10293    update_snapshot(snapshot, Some(module_id), |state| {
10294        state.state = ModuleState::Draining;
10295        state.draining_to_replace = final_state == ModuleState::Restarting;
10296        if let Some(enabled) = enabled {
10297            state.enabled = enabled;
10298        }
10299    })?;
10300
10301    // The wait below is the same budget in every case; what differs is
10302    // whether anything has ASKED the child to stop before it starts. Only a
10303    // forwarding drain that reached the module's registered connection has
10304    // (`module.draining`, then a module GOODBYE). Every other child was told
10305    // nothing: a `protocol: "none"` module, which never registers; a subc
10306    // module spawned moments ago that has not sent HELLO yet; or a stop that
10307    // runs no forwarding drain. Without a signal the budget is only a delay
10308    // in front of SIGKILL -- and the not-yet-registered child is the worst
10309    // case, because it registers into a module that is already draining,
10310    // is never told, and is killed while healthy.
10311    if stop_notice != StopNotice::SentOverConnection {
10312        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10313            info!(
10314                module_id,
10315                pid = child.pid,
10316                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10317                "module has no connection yet; requesting stop by signal"
10318            );
10319        }
10320        request_graceful_stop(module_id, &child);
10321    }
10322
10323    let exit_report = match timeout(drain_timeout, child.wait()).await {
10324        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10325        Ok(Err(source)) => {
10326            fail_after_wait_error(
10327                module_id,
10328                snapshot,
10329                terminal_ring,
10330                spawn_events,
10331                format!("failed to wait during drain: {source}"),
10332            );
10333            return Err(SuperviseError::Wait {
10334                module_id: module_id.to_string(),
10335                source,
10336            });
10337        }
10338        Err(_) => {
10339            // Mirror the sibling arm above: state is already `Draining`, and an
10340            // error propagated from here would strand it there -- a state
10341            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
10342            // `Failed | Stopped`), leaving an operator Restart as the only exit.
10343            // `Failed` before `?` keeps the module operator-visible and
10344            // revivable. Trigger is an ESRCH race (process exits between the
10345            // drain timeout firing and the kill) or a post-kill wait failure
10346            // (issue #34).
10347            //
10348            // Logged because the kill is otherwise visible only as signal 9 in
10349            // the terminal ring, and the budget it follows can be long enough
10350            // that consumers see a stretch of refusals with no stated cause.
10351            warn!(
10352                module_id,
10353                pid = child.pid,
10354                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10355                reason = ?final_state,
10356                ?stop_notice,
10357                "drain budget expired before the module exited; killing it"
10358            );
10359            child.start_kill().map_err(|source| {
10360                fail_after_wait_error(
10361                    module_id,
10362                    snapshot,
10363                    terminal_ring,
10364                    spawn_events,
10365                    format!("failed to kill during drain: {source}"),
10366                );
10367                SuperviseError::Kill {
10368                    module_id: module_id.to_string(),
10369                    source,
10370                }
10371            })?;
10372            let status = child.wait().await.map_err(|source| {
10373                fail_after_wait_error(
10374                    module_id,
10375                    snapshot,
10376                    terminal_ring,
10377                    spawn_events,
10378                    format!("failed to wait after drain kill: {source}"),
10379                );
10380                SuperviseError::Wait {
10381                    module_id: module_id.to_string(),
10382                    source,
10383                }
10384            })?;
10385            classify_reaped_child_exit(snapshot, &child, &status)
10386        }
10387    };
10388
10389    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10390    publish_after_terminal_record(
10391        module_id,
10392        || {
10393            record_terminal_with_detail(
10394                module_id,
10395                terminal_ring,
10396                spawn_events,
10397                &exit_report,
10398                terminal_disposition(final_state),
10399                detail,
10400            )
10401        },
10402        || {
10403            update_snapshot(snapshot, Some(module_id), |state| {
10404                state.state = final_state;
10405                if let Some(enabled) = enabled {
10406                    state.enabled = enabled;
10407                }
10408                clear_current_process_facts(state);
10409                state.last_exit = Some(exit_report.clone());
10410                if exit_report.kind == ExitKind::DeliberateSeverance {
10411                    state.lifetime_restarts += 1;
10412                }
10413            })
10414        },
10415    )?;
10416    child.drain_stderr(module_id).await;
10417
10418    release_dead_registration(registry, forwarding, snapshot, module_id).await
10419}
10420
10421/// Ask a child that nothing else has asked to stop, by signal.
10422///
10423/// A registered subc module is asked over its own connection: the drain sends
10424/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
10425/// module GOODBYE, and the module stops itself. A module that speaks no subc
10426/// wire receives none of that, and neither does a subc module that has not
10427/// registered yet, so for them the drain budget would be pure delay in front of
10428/// a SIGKILL -- and for a process with a store to flush (JetStream is the
10429/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
10430/// into a recovery on the next start.
10431///
10432/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
10433/// rule rather than an optimisation: that module's graceful stop is already
10434/// running by the time its child is drained, and a signal would race it.
10435///
10436/// Best-effort by construction. A child that has already exited is the ordinary
10437/// case rather than an error (the kill lands on a reaped or exiting pid), so a
10438/// failure is logged at debug and the wait-then-kill below still decides the
10439/// outcome.
10440#[cfg(unix)]
10441fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10442    let Some(pid) = child
10443        .id()
10444        .and_then(|pid| i32::try_from(pid).ok())
10445        .and_then(rustix::process::Pid::from_raw)
10446    else {
10447        debug!(
10448            module_id,
10449            "no pid to signal for teardown; falling through to the drain wait"
10450        );
10451        return;
10452    };
10453    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10454        Ok(()) => debug!(
10455            module_id,
10456            "sent SIGTERM to a module nothing else asked to stop"
10457        ),
10458        Err(err) => debug!(
10459            module_id,
10460            error = %err,
10461            "SIGTERM to module failed; the drain wait and kill still apply"
10462        ),
10463    }
10464}
10465
10466/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
10467/// Windows does offer need cooperation this supervisor cannot assume: a console
10468/// control event requires sharing a console with the child, and `WM_CLOSE`
10469/// requires the child to pump a message loop. A supervised server process does
10470/// neither, so there is nothing to send and teardown is the wait followed by the
10471/// kill. Emulating a signal here would mean inventing a stop protocol, which is
10472/// the thing `protocol: "none"` exists to avoid.
10473#[cfg(not(unix))]
10474fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
10475    debug!(
10476        module_id,
10477        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
10478    );
10479}
10480
10481fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
10482    match final_state {
10483        ModuleState::Stopped => TerminalDisposition::Stopped,
10484        ModuleState::Disabled => TerminalDisposition::Disabled,
10485        ModuleState::Restarting => TerminalDisposition::Restarting,
10486        ModuleState::Failed => TerminalDisposition::Failed,
10487        ModuleState::Starting
10488        | ModuleState::Running
10489        | ModuleState::Unresponsive
10490        | ModuleState::Draining => {
10491            unreachable!("terminal exits only finish in terminal or restarting states")
10492        }
10493    }
10494}
10495
10496/// Release a reaped child's registration before allowing another spawn.
10497///
10498/// EOF is not a process-lifetime signal: an inherited socket can stay open
10499/// indefinitely, and serial frame dispatch can be waiting on egress instead of
10500/// reading EOF. After the normal release grace, request connection close (which
10501/// cancels both reads and dispatch), then allow one more release grace for the
10502/// connection guard's forwarding cleanup. Never evict a different connection.
10503async fn release_dead_registration(
10504    registry: &Registry,
10505    forwarding: Option<&ForwardingTable>,
10506    snapshot: &SharedSnapshot,
10507    module_id: &str,
10508) -> Result<(), SuperviseError> {
10509    let result = async {
10510        let registration = registry
10511            .get_module(module_id)
10512            .map_err(SuperviseError::Registry)?;
10513        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
10514            Ok(()) => return Ok(()),
10515            Err(SuperviseError::RegistrationStillActive { .. }) => {}
10516            Err(err) => return Err(err),
10517        }
10518        let pid = lock_snapshot(snapshot)?.reaped_pid;
10519        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
10520            warn!(
10521                module_id,
10522                pid,
10523                connection_id = registration.connection_id.get(),
10524                "reaped module registration outlived release grace; closing dead connection"
10525            );
10526            forwarding.request_connection_close(
10527                registration.connection_id,
10528                CloseReason::new(
10529                    "supervised_process_reaped",
10530                    format!("module '{module_id}' pid {pid} exited"),
10531                ),
10532            );
10533            wait_for_slot_registration_release(
10534                registry,
10535                crate::registry::RegistrationSlot::Connection(registration.connection_id),
10536                REGISTRY_RELEASE_TIMEOUT,
10537            )
10538            .await?;
10539        }
10540        wait_for_registration_release(registry, module_id, Duration::ZERO).await
10541    }
10542    .await;
10543    if let Err(err) = &result {
10544        fail_snapshot(snapshot, Some(module_id), None);
10545        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
10546    }
10547    result
10548}
10549
10550/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
10551/// plain stop or restart waits for before it spawns a replacement.
10552async fn wait_for_registration_release(
10553    registry: &Registry,
10554    module_id: &str,
10555    wait: Duration,
10556) -> Result<(), SuperviseError> {
10557    wait_for_slot_registration_release(
10558        registry,
10559        crate::registry::RegistrationSlot::Active(module_id),
10560        wait,
10561    )
10562    .await
10563}
10564
10565/// Wait for the registration in `slot` to go away.
10566///
10567/// Keyed on the slot rather than the bare module id because a successful swap
10568/// never empties the id's active slot (the promoted candidate is in it), so an
10569/// id-keyed wait for the incumbent's release would always time out. Draining a
10570/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
10571/// incumbent's connection instead.
10572async fn wait_for_slot_registration_release(
10573    registry: &Registry,
10574    slot: crate::registry::RegistrationSlot<'_>,
10575    wait: Duration,
10576) -> Result<(), SuperviseError> {
10577    let deadline = Instant::now() + wait;
10578    let mut release_events = registration_release_events().subscribe();
10579    let still_active = |registration: &crate::registry::ModuleRegistration| {
10580        SuperviseError::RegistrationStillActive {
10581            module_id: registration.manifest.module_id.clone(),
10582            waited: wait,
10583        }
10584    };
10585    loop {
10586        let _observed_generation = *release_events.borrow_and_update();
10587        let Some(registration) = registry
10588            .registration(slot)
10589            .map_err(SuperviseError::Registry)?
10590        else {
10591            return Ok(());
10592        };
10593
10594        let now = Instant::now();
10595        if now >= deadline {
10596            return Err(still_active(&registration));
10597        }
10598
10599        let remaining = deadline.saturating_duration_since(now);
10600        match timeout(remaining, release_events.changed()).await {
10601            Ok(Ok(())) | Ok(Err(_)) => {}
10602            Err(_) => return Err(still_active(&registration)),
10603        }
10604    }
10605}
10606
10607#[cfg(test)]
10608mod slot_registration_wait_tests {
10609    use super::*;
10610    use crate::registry::{ConnectionId, RegistrationSlot};
10611    use subc_protocol::manifest::ModuleManifest;
10612
10613    #[tokio::test]
10614    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
10615        let registry = Arc::new(Registry::default());
10616        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
10617        let runtime = supervisor.runtime_config();
10618        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
10619        let spec = ModuleSpec {
10620            module_id: "enable-stale-registration".to_string(),
10621            program: PathBuf::from("/missing/enable-retry-test"),
10622            args: Vec::new(),
10623            env: Vec::new(),
10624            reserved: false,
10625            reserved_prefixes: Vec::new(),
10626            protocol: ModuleProtocol::Subc,
10627            overlap: Default::default(),
10628        };
10629        let connection = ConnectionId::new(90);
10630        registry
10631            .register_with_control_ops(
10632                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
10633                1,
10634                connection,
10635                Vec::new(),
10636            )
10637            .unwrap();
10638        let mut child = None;
10639        let err = set_child_enabled(
10640            &spec,
10641            &runtime,
10642            &registry,
10643            &supervisor.process_liveness,
10644            &snapshot,
10645            &mut child,
10646            true,
10647        )
10648        .await
10649        .unwrap_err();
10650        assert!(matches!(
10651            err,
10652            SuperviseError::RegistrationStillActive { .. }
10653        ));
10654        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10655        assert!(child.is_none());
10656        registry.deregister_connection(connection).unwrap();
10657        let err = set_child_enabled(
10658            &spec,
10659            &runtime,
10660            &registry,
10661            &supervisor.process_liveness,
10662            &snapshot,
10663            &mut child,
10664            true,
10665        )
10666        .await
10667        .unwrap_err();
10668        assert!(
10669            matches!(err, SuperviseError::Spawn { .. }),
10670            "second enable must attempt a spawn: {err}"
10671        );
10672        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10673    }
10674
10675    const INCUMBENT: u64 = 1;
10676    const CANDIDATE: u64 = 2;
10677
10678    fn swapped_registry() -> Arc<Registry> {
10679        let registry = Arc::new(Registry::default());
10680        let manifest = ModuleManifest::builder("m", "0.1.0").build();
10681        registry
10682            .register_with_control_ops(
10683                manifest.clone(),
10684                1,
10685                ConnectionId::new(INCUMBENT),
10686                Vec::new(),
10687            )
10688            .unwrap();
10689        registry
10690            .register_candidate_with_control_ops(
10691                manifest,
10692                1,
10693                ConnectionId::new(CANDIDATE),
10694                Vec::new(),
10695            )
10696            .unwrap();
10697        registry
10698    }
10699
10700    /// After a promotion the id's active slot is held by the new process, so an
10701    /// id-keyed wait for the incumbent's release can never succeed; the
10702    /// connection-keyed wait completes as soon as the incumbent deregisters.
10703    #[tokio::test]
10704    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
10705        let registry = swapped_registry();
10706        registry.promote_candidate("m").unwrap().unwrap();
10707
10708        assert!(matches!(
10709            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
10710            Err(SuperviseError::RegistrationStillActive { .. })
10711        ));
10712
10713        // Still held while the incumbent's connection has not deregistered.
10714        assert!(matches!(
10715            wait_for_slot_registration_release(
10716                &registry,
10717                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
10718                Duration::from_millis(50),
10719            )
10720            .await,
10721            Err(SuperviseError::RegistrationStillActive { .. })
10722        ));
10723
10724        let releaser = Arc::clone(&registry);
10725        let release = tokio::spawn(async move {
10726            sleep(Duration::from_millis(20)).await;
10727            releaser
10728                .deregister_connection(ConnectionId::new(INCUMBENT))
10729                .unwrap();
10730            notify_registration_release();
10731        });
10732        wait_for_slot_registration_release(
10733            &registry,
10734            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
10735            Duration::from_secs(5),
10736        )
10737        .await
10738        .expect("the incumbent's own registration is released");
10739        release.await.unwrap();
10740        assert!(registry.get_module("m").unwrap().is_some());
10741    }
10742
10743    /// The candidate slot is waited on separately from the active slot: the
10744    /// incumbent's registration neither holds up nor stands in for it.
10745    #[tokio::test]
10746    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
10747        let registry = swapped_registry();
10748        assert!(matches!(
10749            wait_for_slot_registration_release(
10750                &registry,
10751                RegistrationSlot::Candidate("m"),
10752                Duration::from_millis(50),
10753            )
10754            .await,
10755            Err(SuperviseError::RegistrationStillActive { .. })
10756        ));
10757        registry
10758            .deregister_connection(ConnectionId::new(CANDIDATE))
10759            .unwrap();
10760        wait_for_slot_registration_release(
10761            &registry,
10762            RegistrationSlot::Candidate("m"),
10763            Duration::from_millis(50),
10764        )
10765        .await
10766        .expect("a candidate slot with no candidate is released");
10767        assert!(registry
10768            .registration(RegistrationSlot::Active("m"))
10769            .unwrap()
10770            .is_some());
10771    }
10772}
10773
10774fn classify_exit(status: &ExitStatus) -> ExitReport {
10775    ExitReport {
10776        kind: if status.success() {
10777            ExitKind::Clean
10778        } else {
10779            ExitKind::Crash
10780        },
10781        code: status.code(),
10782        signal: exit_signal(status),
10783        at_ms: unix_ms_now(),
10784    }
10785}
10786
10787/// The terminal record for a module whose `wait()` call itself errored (e.g. the
10788/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
10789/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
10790/// disposition still must be `Failed` so the terminal ring is not silently missing
10791/// an entry, matching what `fail_snapshot` records for this same arm.
10792fn wait_error_exit_report() -> ExitReport {
10793    ExitReport {
10794        kind: ExitKind::Crash,
10795        code: None,
10796        signal: None,
10797        at_ms: unix_ms_now(),
10798    }
10799}
10800
10801#[cfg(unix)]
10802fn exit_signal(status: &ExitStatus) -> Option<i32> {
10803    use std::os::unix::process::ExitStatusExt;
10804
10805    status.signal()
10806}
10807
10808#[cfg(not(unix))]
10809fn exit_signal(_status: &ExitStatus) -> Option<i32> {
10810    None
10811}
10812
10813/// Give an operator-touched module its full crash budget back.
10814///
10815/// Named for the counter it used to zero; it now empties the in-window ring,
10816/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
10817/// ledger of what happened survives every operator action.
10818fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
10819    update_snapshot(snapshot, Some(module_id), |state| {
10820        state.clear_crash_restarts();
10821    })
10822}
10823
10824fn set_running(
10825    snapshot: &SharedSnapshot,
10826    child: &SupervisedChild,
10827    module_id: &str,
10828    spawn_events: &SpawnEventFeed,
10829) -> Result<(), SuperviseError> {
10830    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
10831        module_id: Some(module_id.to_string()),
10832    })?;
10833    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
10834    if std::mem::take(&mut state.coalesced_restart_pending) {
10835        let generation = state.spawn_generation;
10836        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
10837    }
10838    state.drain_disposition_detail = None;
10839    state.spawn_failure = None;
10840    state.waiting_for_previous_pid = None;
10841    state.waiting_for_previous_detail = None;
10842    // Every caller of this is a plain spawn, which always uses the primary key;
10843    // a promoted swap candidate sets the flag itself after this returns.
10844    state.in_alternate_slot = false;
10845    state.configuration_updated_since_spawn = false;
10846    state.spawned_protocol = Some(child.protocol);
10847    state.state = ModuleState::Running;
10848    state.enabled = true;
10849    state.process_alive = true;
10850    state.pid = child.id();
10851    #[cfg(windows)]
10852    {
10853        state.pipe_consumed = child
10854            .nonce_delivery
10855            .as_ref()
10856            .map(|delivery| delivery.consumed());
10857    }
10858    #[cfg(target_os = "macos")]
10859    {
10860        state.report_ready = Some(Arc::clone(&child.report_ready));
10861    }
10862    state.spawned_at_ms = Some(child.spawned_at_ms);
10863    state.exec_outcome = Some(Arc::clone(&child.exec.0));
10864    state.spawned_from = Some(child.spawned_from.clone());
10865    state.spawned_file_identity = child.spawned_file_identity.clone();
10866    state.process_start_time = child.process_start_time;
10867    Ok(())
10868}
10869
10870fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
10871    state.waiting_for_previous_pid = None;
10872    state.waiting_for_previous_detail = None;
10873    state.process_alive = false;
10874    state.spawned_protocol = None;
10875    state.pid = None;
10876    #[cfg(windows)]
10877    {
10878        state.pipe_consumed = None;
10879    }
10880    #[cfg(target_os = "macos")]
10881    {
10882        state.report_ready = None;
10883    }
10884    state.spawned_at_ms = None;
10885    state.exec_outcome = None;
10886    state.spawned_from = None;
10887    state.spawned_file_identity = None;
10888    state.process_start_time = None;
10889    state.deliberate_severance = None;
10890}
10891
10892#[cfg(test)]
10893fn record_deliberate_severance(
10894    snapshot: &SharedSnapshot,
10895    identity: ProcessIdentity,
10896) -> Result<(), SuperviseError> {
10897    update_snapshot(snapshot, None, |state| {
10898        state.deliberate_severance = Some(identity);
10899    })
10900}
10901
10902fn apply_deliberate_severance_marker(
10903    snapshot: &SharedSnapshot,
10904    exited_identity: Option<ProcessIdentity>,
10905    mut exit_report: ExitReport,
10906) -> ExitReport {
10907    let marker = lock_snapshot(snapshot)
10908        .ok()
10909        .and_then(|mut state| state.deliberate_severance.take());
10910    if marker.is_some() && marker == exited_identity {
10911        exit_report.kind = ExitKind::DeliberateSeverance;
10912    }
10913    exit_report
10914}
10915
10916fn classify_reaped_child_exit(
10917    snapshot: &SharedSnapshot,
10918    child: &SupervisedChild,
10919    status: &ExitStatus,
10920) -> ExitReport {
10921    let _ = update_snapshot(snapshot, None, |state| {
10922        state.reaped_pid = Some(child.pid);
10923        state.spawn_failure = child.spawn_failure.clone();
10924    });
10925    apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
10926}
10927
10928fn fail_snapshot(
10929    snapshot: &SharedSnapshot,
10930    module_id: Option<&str>,
10931    last_exit: Option<ExitReport>,
10932) {
10933    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
10934        state.state = ModuleState::Failed;
10935        clear_current_process_facts(state);
10936        if let Some(last_exit) = last_exit {
10937            state.last_exit = Some(last_exit);
10938        }
10939    }) {
10940        error!(error = %err, "failed to mark supervisor state failed");
10941    }
10942}
10943
10944fn update_snapshot(
10945    snapshot: &SharedSnapshot,
10946    module_id: Option<&str>,
10947    update: impl FnOnce(&mut SupervisorSnapshot),
10948) -> Result<(), SuperviseError> {
10949    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
10950        module_id: module_id.map(ToOwned::to_owned),
10951    })?;
10952    update(&mut state);
10953    Ok(())
10954}
10955
10956const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
10957
10958fn lock_snapshot_for_control<'a>(
10959    snapshot: &'a SharedSnapshot,
10960    module_id: &str,
10961    caller: &'static str,
10962) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
10963    let started_at = Instant::now();
10964    let guard = lock_snapshot(snapshot)?;
10965    let waited = started_at.elapsed();
10966    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
10967        warn!(
10968            module_id = %module_id,
10969            waited_ms = waited.as_millis() as u64,
10970            caller = %caller,
10971            "slow snapshot lock"
10972        );
10973    }
10974    Ok(guard)
10975}
10976
10977fn lock_snapshot(
10978    snapshot: &SharedSnapshot,
10979) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
10980    snapshot
10981        .lock()
10982        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
10983}
10984
10985#[cfg(test)]
10986mod terminal_history_tests {
10987    use std::{
10988        path::PathBuf,
10989        sync::Arc,
10990        time::{Duration, Instant},
10991    };
10992
10993    use tokio::{sync::mpsc, time::sleep};
10994
10995    use super::{
10996        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
10997        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
10998        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
10999        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11000        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11001        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11002        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11003        SupervisorHealthStatus, SupervisorSnapshot,
11004    };
11005    // The supervisor's clock, distinct from the `std::time::Instant` these tests
11006    // use for their own wall-clock deadlines: crash-restart instants must be on
11007    // the same clock the production code stamps them with, which is tokio's (and
11008    // is what `start_paused` tests can move).
11009    use super::Instant as ClockInstant;
11010    use crate::{
11011        registry::Registry,
11012        terminal_ring::{TerminalRing, TerminalRingConfig},
11013    };
11014    use std::sync::Mutex;
11015    use subc_control::TerminalDisposition;
11016
11017    /// See the twin in `control.rs` for why this derives the path from
11018    /// `current_exe()` and why the existence check is here: `--lib` alone does
11019    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
11020    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
11021    pub(super) fn fake_aft_stub_path() -> PathBuf {
11022        let mut path = std::env::current_exe().expect("current_exe available in tests");
11023        path.pop();
11024        path.pop();
11025        path.push(if cfg!(windows) {
11026            "fake-aft-stub.exe"
11027        } else {
11028            "fake-aft-stub"
11029        });
11030        assert!(
11031            path.exists(),
11032            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11033             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11034            path.display()
11035        );
11036        path
11037    }
11038
11039    #[test]
11040    fn reserved_never_spawned_refuses_every_hello() {
11041        // The canary hole: a reserved id whose module has never spawned had NO
11042        // gate entry and admitted anyone -- the reservation protected the nonce
11043        // holder, not the NAME. Now the entry is present with no legitimate
11044        // holder and refuses all comers.
11045        let supervisor = SupervisorHandle::default();
11046        supervisor.apply_identity_configuration(&ModuleSpec {
11047            module_id: "never-spawned".to_string(),
11048            program: PathBuf::from("/usr/bin/false"),
11049            args: Vec::new(),
11050            env: Vec::new(),
11051            reserved: true,
11052            reserved_prefixes: Vec::new(),
11053            protocol: ModuleProtocol::Subc,
11054            overlap: Default::default(),
11055        });
11056        assert!(
11057            supervisor
11058                .reserved_hello_rejection("never-spawned", Some("any-forged-nonce"))
11059                .is_some(),
11060            "forged nonce must refuse on a reserved never-spawned id"
11061        );
11062        assert!(
11063            supervisor
11064                .reserved_hello_rejection("never-spawned", None)
11065                .is_some(),
11066            "absent nonce must refuse on a reserved never-spawned id"
11067        );
11068        // And a real spawn nonce minted later admits exactly that nonce.
11069        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11070        supervisor.apply_identity_configuration(&ModuleSpec {
11071            module_id: "never-spawned".to_string(),
11072            program: PathBuf::from("/usr/bin/false"),
11073            args: Vec::new(),
11074            env: Vec::new(),
11075            reserved: true,
11076            reserved_prefixes: Vec::new(),
11077            protocol: ModuleProtocol::Subc,
11078            overlap: Default::default(),
11079        });
11080        assert!(supervisor
11081            .reserved_hello_rejection("never-spawned", Some("minted"))
11082            .is_none());
11083        assert!(supervisor
11084            .reserved_hello_rejection("never-spawned", Some("forged"))
11085            .is_some());
11086    }
11087
11088    /// Put `count` crash restarts on a snapshot's ring as if they had all just
11089    /// happened, which is what "spent budget" looks like to every reader.
11090    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11091        let now = ClockInstant::now();
11092        for _ in 0..count {
11093            state.crash_restarts.push_back(now);
11094        }
11095    }
11096
11097    /// Age the oldest recorded restart out of `window`, standing in for the hours
11098    /// that would otherwise have to pass. Injecting the instant is the point: a
11099    /// test that slept a real window would take ten minutes and still prove less.
11100    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11101        let aged = state
11102            .crash_restarts
11103            .front()
11104            .expect("a crash restart must be recorded before it can be aged")
11105            .checked_sub(window + Duration::from_secs(1))
11106            .expect("the test clock is far enough from its origin to age an instant");
11107        state.crash_restarts[0] = aged;
11108    }
11109
11110    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11111        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11112        seed_crash_restarts(&mut state, count);
11113        state
11114    }
11115
11116    #[test]
11117    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11118        let policy = RestartPolicy::new(3, Duration::ZERO);
11119        let now = ClockInstant::now();
11120        assert!(daemon_will_restart(
11121            &mut snapshot_with_restarts(true, 2),
11122            &policy,
11123            now
11124        ));
11125        assert!(!daemon_will_restart(
11126            &mut snapshot_with_restarts(true, 3),
11127            &policy,
11128            now
11129        ));
11130        assert!(!daemon_will_restart(
11131            &mut snapshot_with_restarts(false, 0),
11132            &policy,
11133            now
11134        ));
11135    }
11136
11137    #[test]
11138    fn crash_restart_backoff_escalates_with_in_window_count() {
11139        let policy = RestartPolicy::new(4, Duration::from_millis(100))
11140            .with_max_backoff(Duration::from_secs(30));
11141        let now = ClockInstant::now();
11142        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11143        let schedules = (0..4)
11144            .map(|_| {
11145                state
11146                    .next_crash_restart(&policy, now)
11147                    .expect("the test policy allows four crash restarts")
11148            })
11149            .collect::<Vec<_>>();
11150
11151        assert_eq!(
11152            schedules
11153                .iter()
11154                .map(|schedule| schedule.restart_in_window)
11155                .collect::<Vec<_>>(),
11156            vec![0, 1, 2, 3]
11157        );
11158        assert_eq!(
11159            schedules
11160                .iter()
11161                .map(|schedule| schedule.delay)
11162                .collect::<Vec<_>>(),
11163            vec![
11164                Duration::from_millis(100),
11165                Duration::from_secs(1),
11166                Duration::from_secs(10),
11167                Duration::from_secs(30),
11168            ]
11169        );
11170    }
11171
11172    #[test]
11173    fn crash_restart_backoff_resets_after_ring_clear() {
11174        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11175        let now = ClockInstant::now();
11176        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11177        assert_eq!(
11178            state.next_crash_restart(&policy, now).unwrap().delay,
11179            Duration::from_millis(100)
11180        );
11181        assert_eq!(
11182            state.next_crash_restart(&policy, now).unwrap().delay,
11183            Duration::from_secs(1)
11184        );
11185
11186        state.clear_crash_restarts();
11187        let schedule = state
11188            .next_crash_restart(&policy, now)
11189            .expect("a cleared ring must allow another restart");
11190        assert_eq!(schedule.restart_in_window, 0);
11191        assert_eq!(schedule.delay, Duration::from_millis(100));
11192    }
11193
11194    #[test]
11195    fn crash_restart_backoff_ignores_aged_restarts() {
11196        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11197        let now = ClockInstant::now();
11198        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11199        state
11200            .next_crash_restart(&policy, now)
11201            .expect("the first restart is allowed");
11202        state
11203            .next_crash_restart(&policy, now)
11204            .expect("the second restart is allowed");
11205        state.crash_restarts[0] = now
11206            .checked_sub(policy.window + Duration::from_secs(1))
11207            .expect("the fake clock can age a restart past the window");
11208
11209        let schedule = state
11210            .next_crash_restart(&policy, now)
11211            .expect("an aged restart must release its slot");
11212        assert_eq!(schedule.restart_in_window, 1);
11213        assert_eq!(schedule.delay, Duration::from_secs(1));
11214        assert_eq!(state.crash_restarts.len(), 2);
11215    }
11216
11217    /// The budget is a rate: the same three spent restarts refuse a respawn
11218    /// while they are recent and allow one once they have aged past the window.
11219    /// Nothing about the module changed in between, which is the whole point.
11220    #[test]
11221    fn a_budget_spent_before_the_window_no_longer_refuses() {
11222        let policy = RestartPolicy::new(3, Duration::ZERO);
11223        let mut state = snapshot_with_restarts(true, 3);
11224        let now = ClockInstant::now();
11225        assert!(!daemon_will_restart(&mut state, &policy, now));
11226
11227        assert!(daemon_will_restart(
11228            &mut state,
11229            &policy,
11230            now + policy.window + Duration::from_secs(1)
11231        ));
11232        assert!(
11233            state.crash_restarts.is_empty(),
11234            "reading the budget must drop the instants that left the window"
11235        );
11236    }
11237
11238    fn module_with_recovery_snapshot(
11239        state: ModuleState,
11240        enabled: bool,
11241        restart_count: u32,
11242    ) -> SupervisedModule {
11243        let registry = Arc::new(Registry::default());
11244        let supervisor =
11245            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
11246        let runtime = supervisor.runtime_config();
11247        let spec = ModuleSpec {
11248            module_id: "recovery-snapshot".to_string(),
11249            program: fake_aft_stub_path(),
11250            args: Vec::new(),
11251            env: Vec::new(),
11252            reserved: false,
11253            reserved_prefixes: Vec::new(),
11254            protocol: ModuleProtocol::Subc,
11255            overlap: Default::default(),
11256        };
11257        let mut snapshot = SupervisorSnapshot::new(state, enabled);
11258        seed_crash_restarts(&mut snapshot, restart_count);
11259        // These tests read synthetic snapshots. A real child and monitor would
11260        // race those reads by replacing the requested state during startup.
11261        let (commands, _rx) = mpsc::channel(4);
11262        SupervisedModule {
11263            inner: Arc::new(SupervisedModuleInner {
11264                module_id: spec.module_id.clone(),
11265                registry,
11266                snapshot: Arc::new(Mutex::new(snapshot)),
11267                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11268                    spec,
11269                    health: runtime.health,
11270                })),
11271                stderr_ring: runtime.stderr_ring,
11272                terminal_ring: runtime.terminal_ring,
11273                commands,
11274                monitor: Mutex::new(None),
11275                restart_policy: runtime.restart_policy,
11276                effective_drain_timeout: runtime.effective_drain_timeout,
11277                provenance_probe: supervisor.provenance_probe.clone(),
11278            }),
11279        }
11280    }
11281
11282    #[cfg(target_os = "linux")]
11283    #[tokio::test]
11284    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11285        let supervisor =
11286            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11287                .with_cgroup_placement(None);
11288        let result = supervisor.spawn(ModuleSpec {
11289            module_id: "no-cgroup-placement".to_string(),
11290            program: fake_aft_stub_path(),
11291            args: Vec::new(),
11292            env: Vec::new(),
11293            reserved: false,
11294            reserved_prefixes: Vec::new(),
11295            protocol: ModuleProtocol::Subc,
11296            overlap: Default::default(),
11297        });
11298
11299        assert!(
11300            result.is_ok(),
11301            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11302        );
11303    }
11304
11305    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11306    async fn undecided_snapshot_uses_shared_restart_predicate() {
11307        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11308            .will_recover_after_connection_loss()
11309            .unwrap());
11310        assert!(
11311            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11312                .will_recover_after_connection_loss()
11313                .unwrap()
11314        );
11315    }
11316
11317    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11318    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11319        assert!(
11320            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11321                .will_recover_after_connection_loss()
11322                .unwrap()
11323        );
11324    }
11325
11326    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11327    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11328        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11329            .will_recover_after_connection_loss()
11330            .unwrap());
11331        assert!(
11332            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11333                .will_recover_after_connection_loss()
11334                .unwrap()
11335        );
11336    }
11337
11338    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11339    async fn warming_snapshot_is_limited_to_startup_phases() {
11340        for state in [
11341            ModuleState::Starting,
11342            ModuleState::Running,
11343            ModuleState::Restarting,
11344        ] {
11345            assert!(
11346                module_with_recovery_snapshot(state, true, 0)
11347                    .is_warming()
11348                    .unwrap(),
11349                "{state:?} should be warming"
11350            );
11351        }
11352        for state in [
11353            ModuleState::Unresponsive,
11354            ModuleState::Draining,
11355            ModuleState::Stopped,
11356            ModuleState::Failed,
11357            ModuleState::Disabled,
11358        ] {
11359            assert!(
11360                !module_with_recovery_snapshot(state, true, 0)
11361                    .is_warming()
11362                    .unwrap(),
11363                "{state:?} should not be warming"
11364            );
11365        }
11366    }
11367
11368    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11369    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11370        let registry = Arc::new(Registry::default());
11371        let supervisor =
11372            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
11373        let module = supervisor
11374            .spawn(ModuleSpec {
11375                module_id: "terminal-history".to_string(),
11376                program: fake_aft_stub_path(),
11377                args: Vec::new(),
11378                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11379                reserved: false,
11380                reserved_prefixes: Vec::new(),
11381                protocol: ModuleProtocol::Subc,
11382                overlap: Default::default(),
11383            })
11384            .unwrap();
11385
11386        let deadline = Instant::now() + Duration::from_secs(5);
11387        loop {
11388            let history = module.terminal_history();
11389            if history.entries.len() == 2 {
11390                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11391                assert_eq!(history.dropped, 0);
11392                assert_eq!(
11393                    history
11394                        .entries
11395                        .iter()
11396                        .map(|entry| entry.exit_code)
11397                        .collect::<Vec<_>>(),
11398                    vec![Some(23), Some(23)]
11399                );
11400                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11401                return;
11402            }
11403            assert!(
11404                Instant::now() < deadline,
11405                "module did not retain two terminal exits: {history:?}"
11406            );
11407            sleep(Duration::from_millis(10)).await;
11408        }
11409    }
11410
11411    /// A disable issued while a crash respawn is still backing off must preempt
11412    /// that respawn: the operator's stop wins, the disable must not queue behind
11413    /// the backoff, and the module must never come back up afterwards.
11414    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11415    async fn disable_during_crash_backoff_cancels_pending_respawn() {
11416        let backoff = Duration::from_secs(2);
11417        let supervisor = Supervisor::new_for_test(
11418            Arc::new(Registry::default()),
11419            RestartPolicy::new(10, backoff),
11420        );
11421        let module = supervisor
11422            .spawn(ModuleSpec {
11423                module_id: "disable-during-backoff".to_string(),
11424                program: fake_aft_stub_path(),
11425                args: Vec::new(),
11426                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11427                reserved: false,
11428                reserved_prefixes: Vec::new(),
11429                protocol: ModuleProtocol::Subc,
11430                overlap: Default::default(),
11431            })
11432            .unwrap();
11433
11434        // Wait for the first crash to put the module into its backoff window.
11435        let deadline = Instant::now() + Duration::from_secs(5);
11436        loop {
11437            if module.status().unwrap().state == ModuleState::Restarting {
11438                break;
11439            }
11440            assert!(
11441                Instant::now() < deadline,
11442                "module never entered the crash backoff"
11443            );
11444            sleep(Duration::from_millis(10)).await;
11445        }
11446
11447        let started = Instant::now();
11448        module.set_enabled(false).await.unwrap();
11449        let waited = started.elapsed();
11450
11451        assert!(
11452            waited < backoff / 2,
11453            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
11454        );
11455        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
11456
11457        // Outlast the backoff: the respawn it was counting down to must never run.
11458        sleep(backoff + Duration::from_millis(500)).await;
11459        let status = module.status().unwrap();
11460        assert_eq!(status.state, ModuleState::Disabled);
11461        assert_eq!(
11462            status.spawn_generation, 1,
11463            "module respawned after the operator disabled it"
11464        );
11465    }
11466
11467    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
11468    /// the shape of nats-server, the program this rule exists for.
11469    #[cfg(unix)]
11470    fn protocol_none_sigterm_exits_clean_spec(
11471        module_id: &str,
11472        dir: &std::path::Path,
11473    ) -> (ModuleSpec, PathBuf, PathBuf) {
11474        let ready = dir.join("ready");
11475        let marker = dir.join("sigterm");
11476        let spec = ModuleSpec {
11477            module_id: module_id.to_string(),
11478            program: fake_aft_stub_path(),
11479            args: Vec::new(),
11480            env: vec![
11481                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
11482                (
11483                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
11484                    marker.display().to_string(),
11485                ),
11486                (
11487                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
11488                    ready.display().to_string(),
11489                ),
11490            ],
11491            reserved: false,
11492            reserved_prefixes: Vec::new(),
11493            protocol: ModuleProtocol::None,
11494            overlap: Default::default(),
11495        };
11496        (spec, ready, marker)
11497    }
11498
11499    /// Wait for a file the child writes, so a signal is never sent before the
11500    /// child's SIGTERM handler is installed (the default disposition would
11501    /// kill it by signal and the exit would not be clean).
11502    #[cfg(unix)]
11503    async fn wait_for_file(path: &std::path::Path) {
11504        let deadline = Instant::now() + Duration::from_secs(10);
11505        while !path.exists() {
11506            assert!(
11507                Instant::now() < deadline,
11508                "{} never appeared",
11509                path.display()
11510            );
11511            sleep(Duration::from_millis(10)).await;
11512        }
11513    }
11514
11515    /// A protocol-none module that exits 0 because something OUTSIDE the
11516    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
11517    /// the crash-path disposition rather than `stopped`.
11518    #[cfg(unix)]
11519    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11520    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
11521        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
11522        let (spec, ready, marker) =
11523            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
11524        let supervisor = Supervisor::new_for_test(
11525            Arc::new(Registry::default()),
11526            RestartPolicy::new(3, Duration::ZERO),
11527        );
11528        let module = supervisor.spawn(spec).unwrap();
11529        wait_for_file(&ready).await;
11530        // The ready file proves the child installed its SIGTERM handler, not
11531        // that the supervisor has processed the privacy trampoline's exec
11532        // acknowledgement. On macOS status withholds the pid until then.
11533        let deadline = Instant::now() + Duration::from_secs(10);
11534        let first_pid = loop {
11535            let status = module.status().unwrap();
11536            if status.state == ModuleState::Running {
11537                if let Some(pid) = status.pid {
11538                    break pid;
11539                }
11540            }
11541            assert!(
11542                Instant::now() < deadline,
11543                "a running module must report its pid after exec confirmation: {status:?}"
11544            );
11545            sleep(Duration::from_millis(10)).await;
11546        };
11547
11548        rustix::process::kill_process(
11549            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
11550            rustix::process::Signal::TERM,
11551        )
11552        .unwrap();
11553
11554        let deadline = Instant::now() + Duration::from_secs(10);
11555        let respawned = loop {
11556            let status = module.status().unwrap();
11557            if status.state == ModuleState::Running
11558                && status.pid.is_some_and(|pid| pid != first_pid)
11559            {
11560                break status;
11561            }
11562            assert!(
11563                Instant::now() < deadline,
11564                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
11565            );
11566            sleep(Duration::from_millis(10)).await;
11567        };
11568        assert_eq!(respawned.spawn_generation, 2);
11569        assert!(
11570            marker.exists(),
11571            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
11572        );
11573
11574        let history = module.terminal_history();
11575        assert_eq!(history.entries.len(), 1, "{history:?}");
11576        let entry = &history.entries[0];
11577        assert_eq!(entry.exit_code, Some(0));
11578        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
11579        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
11580
11581        module.stop().await.unwrap();
11582    }
11583
11584    /// Repeated unrequested clean exits of a protocol-none module spend the
11585    /// restart budget exactly as crashes do, and the module ends `failed` with
11586    /// the budget named.
11587    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11588    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
11589        let supervisor = Supervisor::new_for_test(
11590            Arc::new(Registry::default()),
11591            RestartPolicy::new(1, Duration::ZERO),
11592        );
11593        let module = supervisor
11594            .spawn(ModuleSpec {
11595                module_id: "none-clean-exit-budget".to_string(),
11596                program: fake_aft_stub_path(),
11597                args: Vec::new(),
11598                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
11599                reserved: false,
11600                reserved_prefixes: Vec::new(),
11601                protocol: ModuleProtocol::None,
11602                overlap: Default::default(),
11603            })
11604            .unwrap();
11605
11606        // Failed follows the terminal write; this deadline only bounds a hang,
11607        // not an assumed duration for the two launches or their exit recording.
11608        let deadline = Instant::now() + Duration::from_secs(10);
11609        loop {
11610            let status = module.status().unwrap();
11611            if status.state == ModuleState::Failed {
11612                break;
11613            }
11614            assert!(
11615                Instant::now() < deadline,
11616                "module never exhausted its budget: {status:?} {:?}",
11617                module.terminal_history()
11618            );
11619            sleep(Duration::from_millis(10)).await;
11620        }
11621        let history = module.terminal_history();
11622        assert_eq!(
11623            history
11624                .entries
11625                .iter()
11626                .map(|entry| (entry.exit_code, entry.disposition.clone()))
11627                .collect::<Vec<_>>(),
11628            vec![
11629                (Some(0), TerminalDisposition::Restarting),
11630                (Some(0), TerminalDisposition::Failed),
11631            ]
11632        );
11633        let detail = history.entries[1]
11634            .disposition_detail
11635            .as_deref()
11636            .expect("a budget failure names the budget");
11637        assert!(detail.contains("max_restarts=1"), "{detail}");
11638        assert_eq!(module.status().unwrap().spawn_generation, 2);
11639    }
11640
11641    #[test]
11642    fn restart_budget_failure_is_published_after_its_terminal_record() {
11643        use std::io::{Read, Write};
11644        let supervisor = Supervisor::new_for_test(
11645            Arc::new(Registry::default()),
11646            RestartPolicy::new(0, Duration::ZERO),
11647        );
11648        let runtime = supervisor.runtime_config();
11649        let spec = ModuleSpec {
11650            protocol: ModuleProtocol::None,
11651            ..windowed_crash_spec("budget-publication-order")
11652        };
11653        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
11654            ModuleState::Running,
11655            true,
11656        )));
11657        lock_snapshot(&snapshot).unwrap().process_alive = true;
11658        let reaped_snapshot = snapshot.clone();
11659        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11660        let history = runtime.terminal_ring.clone();
11661        let reap = std::thread::spawn(move || {
11662            tokio::runtime::Builder::new_current_thread()
11663                .enable_all()
11664                .build()
11665                .unwrap()
11666                .block_on(on_child_exit(
11667                    &spec,
11668                    runtime.restart_policy,
11669                    &supervisor.registry,
11670                    &reaped_snapshot,
11671                    &runtime.terminal_ring,
11672                    &runtime.spawn_events,
11673                    &runtime.child_roster,
11674                    ExitReport {
11675                        kind: ExitKind::Clean,
11676                        code: Some(0),
11677                        signal: None,
11678                        at_ms: 1,
11679                    },
11680                ))
11681        });
11682        let mut reached = [0];
11683        barrier.read_exact(&mut reached).unwrap();
11684        assert_eq!(&reached, b"R");
11685        let (before_state, before_alive) = {
11686            let state = lock_snapshot(&snapshot).unwrap();
11687            (state.state, state.process_alive)
11688        };
11689        let before_history = history.lock().unwrap().snapshot();
11690        barrier.write_all(b"X").unwrap();
11691        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
11692        assert!(before_history.entries.is_empty());
11693        assert_ne!(
11694            before_state,
11695            ModuleState::Failed,
11696            "Failed was visible before its terminal record could be written"
11697        );
11698        assert!(before_alive);
11699        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11700        let history = history.lock().unwrap().snapshot();
11701        assert_eq!(history.entries.len(), 1);
11702        assert_eq!(history.entries[0].exit_code, Some(0));
11703        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11704    }
11705
11706    #[test]
11707    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
11708        use std::io::{Read, Write};
11709
11710        let supervisor = Supervisor::new_for_test(
11711            Arc::new(Registry::default()),
11712            RestartPolicy::new(3, Duration::ZERO),
11713        );
11714        let mut runtime = supervisor.runtime_config();
11715        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
11716        let snapshot = module.inner.snapshot.clone();
11717        lock_snapshot(&snapshot).unwrap().process_alive = true;
11718        runtime.terminal_ring = module.inner.terminal_ring.clone();
11719        let spec = windowed_crash_spec(module.module_id());
11720        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11721        let reaped_snapshot = snapshot.clone();
11722        let reap = std::thread::spawn(move || {
11723            tokio::runtime::Builder::new_current_thread()
11724                .enable_all()
11725                .build()
11726                .unwrap()
11727                .block_on(on_child_exit(
11728                    &spec,
11729                    runtime.restart_policy,
11730                    &supervisor.registry,
11731                    &reaped_snapshot,
11732                    &runtime.terminal_ring,
11733                    &runtime.spawn_events,
11734                    &runtime.child_roster,
11735                    ExitReport {
11736                        kind: ExitKind::Crash,
11737                        code: Some(2),
11738                        signal: None,
11739                        at_ms: 1,
11740                    },
11741                ))
11742        });
11743
11744        // Hold the crash handler between its budget decision and publication.
11745        // Connection cleanup must still see the recovery that this crash chose,
11746        // even when that recovery consumes the last available restart slot.
11747        let mut reached = [0];
11748        barrier.read_exact(&mut reached).unwrap();
11749        assert_eq!(&reached, b"R");
11750        let before_status = module.status().unwrap();
11751        let before_history = module.terminal_history();
11752        let before_recovers = module.will_recover_after_connection_loss().unwrap();
11753        barrier.write_all(b"X").unwrap();
11754        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
11755
11756        assert_eq!(before_status.state, ModuleState::Running);
11757        assert!(before_status.process_alive);
11758        assert_eq!(before_status.restart_count, 3);
11759        assert!(before_history.entries.is_empty());
11760        assert!(
11761            before_recovers,
11762            "connection cleanup would report a terminal close for a scheduled crash restart"
11763        );
11764        let restarting = module.status().unwrap();
11765        assert_eq!(restarting.state, ModuleState::Restarting);
11766        assert_eq!(restarting.restart_count, 3);
11767        assert!(module.will_recover_after_connection_loss().unwrap());
11768        let history = module.terminal_history();
11769        assert_eq!(history.entries.len(), 1);
11770        assert_eq!(
11771            history.entries[0].disposition,
11772            TerminalDisposition::Restarting
11773        );
11774
11775        // Once the replacement is running, its next crash must be judged
11776        // against the exhausted budget rather than the previous recovery.
11777        update_snapshot(&snapshot, Some(module.module_id()), |state| {
11778            state.state = ModuleState::Running;
11779            state.respawn_pending = false;
11780        })
11781        .unwrap();
11782        assert!(!module.will_recover_after_connection_loss().unwrap());
11783        update_snapshot(&snapshot, Some(module.module_id()), |state| {
11784            state.enabled = false;
11785            state.respawn_pending = true;
11786        })
11787        .unwrap();
11788        assert!(!module.will_recover_after_connection_loss().unwrap());
11789    }
11790
11791    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
11792        use std::net::{TcpListener, TcpStream};
11793        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
11794        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
11795        let (barrier, _) = listener.accept().unwrap();
11796        for stream in [&writer, &barrier] {
11797            stream
11798                .set_read_timeout(Some(Duration::from_secs(10)))
11799                .unwrap();
11800            stream
11801                .set_write_timeout(Some(Duration::from_secs(10)))
11802                .unwrap();
11803        }
11804        *events.1.lock().unwrap() = Some(writer);
11805        barrier
11806    }
11807
11808    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
11809        use std::io::{Read, Write};
11810        let barrier = events.1.lock().unwrap().take();
11811        if let Some(mut barrier) = barrier {
11812            barrier.write_all(b"R").unwrap();
11813            let mut release = [0];
11814            barrier.read_exact(&mut release).unwrap();
11815            assert_eq!(&release, b"X");
11816        }
11817    }
11818
11819    #[test]
11820    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
11821        use std::io::{Read, Write};
11822
11823        let supervisor = Supervisor::new_for_test(
11824            Arc::new(Registry::default()),
11825            RestartPolicy::new(0, Duration::ZERO),
11826        );
11827        let mut runtime = supervisor.runtime_config();
11828        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
11829        let snapshot = module.inner.snapshot.clone();
11830        runtime.terminal_ring = module.inner.terminal_ring.clone();
11831        let spec = ModuleSpec {
11832            module_id: module.module_id().into(),
11833            program: fake_aft_stub_path(),
11834            protocol: ModuleProtocol::None,
11835            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
11836            ..windowed_crash_spec(module.module_id())
11837        };
11838        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11839        let reap = std::thread::spawn(move || {
11840            tokio::runtime::Builder::new_current_thread()
11841                .enable_all()
11842                .build()
11843                .unwrap()
11844                .block_on(async {
11845                    let mut child =
11846                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
11847                    health_restart_child(
11848                        &spec,
11849                        &runtime,
11850                        &supervisor.registry,
11851                        &supervisor.process_liveness,
11852                        &snapshot,
11853                        &mut child,
11854                        SupervisorHealthStatus::Failing,
11855                        Some("health probe did not answer"),
11856                        1,
11857                    )
11858                    .await
11859                    .unwrap();
11860                    assert!(child.is_none());
11861                });
11862        });
11863        // The loopback barrier acknowledges entry into terminal recording by
11864        // health_restart_child's drain, then waits for the test's release byte.
11865        // Status and history are observed while that writer is held, not after
11866        // an elapsed-time guess about when the health action has completed.
11867        let mut reached = [0];
11868        barrier.read_exact(&mut reached).unwrap();
11869        assert_eq!(&reached, b"R");
11870        let before_status = module.status().unwrap();
11871        let before_history = module.terminal_history();
11872        barrier.write_all(b"X").unwrap();
11873        reap.join().unwrap();
11874        assert!(before_history.entries.is_empty());
11875        assert_ne!(
11876            before_status.state,
11877            ModuleState::Failed,
11878            "Failed was visible before its terminal record could be written"
11879        );
11880        assert!(
11881            before_status.process_alive,
11882            "process death was visible before its terminal record could be written"
11883        );
11884        let failed = module.status().unwrap();
11885        assert_eq!(failed.state, ModuleState::Failed);
11886        assert!(!failed.process_alive);
11887        let history = module.terminal_history();
11888        assert_eq!(history.entries.len(), 1);
11889        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11890        assert_eq!(
11891            history.entries[0].disposition_detail.as_deref(),
11892            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
11893        );
11894    }
11895
11896    #[test]
11897    fn observed_exits_publish_process_death_after_terminal_history() {
11898        use std::io::{Read, Write};
11899        for (kind, enabled, shutdown, wait_error, expected_state) in [
11900            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
11901            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
11902            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
11903            (
11904                ExitKind::DeliberateSeverance,
11905                true,
11906                false,
11907                false,
11908                ModuleState::Restarting,
11909            ),
11910            #[cfg(unix)]
11911            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
11912            (ExitKind::Crash, true, false, true, ModuleState::Failed),
11913        ] {
11914            let supervisor = Supervisor::new_for_test(
11915                Arc::new(Registry::default()),
11916                RestartPolicy::new(1, Duration::ZERO),
11917            );
11918            let runtime = supervisor.runtime_config();
11919            #[cfg(unix)]
11920            if shutdown {
11921                runtime.child_roster.close();
11922            }
11923            #[cfg(not(unix))]
11924            let _ = shutdown;
11925            let spec = windowed_crash_spec("exit-publication-order");
11926            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
11927                ModuleState::Running,
11928                enabled,
11929            )));
11930            lock_snapshot(&snapshot).unwrap().process_alive = true;
11931            let reaped_snapshot = snapshot.clone();
11932            let history = runtime.terminal_ring.clone();
11933            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11934            let reap = std::thread::spawn(move || {
11935                if wait_error {
11936                    super::fail_after_wait_error(
11937                        &spec.module_id,
11938                        &reaped_snapshot,
11939                        &runtime.terminal_ring,
11940                        &runtime.spawn_events,
11941                        "wait failed".into(),
11942                    );
11943                } else {
11944                    tokio::runtime::Builder::new_current_thread()
11945                        .enable_all()
11946                        .build()
11947                        .unwrap()
11948                        .block_on(on_child_exit(
11949                            &spec,
11950                            runtime.restart_policy,
11951                            &supervisor.registry,
11952                            &reaped_snapshot,
11953                            &runtime.terminal_ring,
11954                            &runtime.spawn_events,
11955                            &runtime.child_roster,
11956                            ExitReport {
11957                                kind,
11958                                code: Some(0),
11959                                signal: None,
11960                                at_ms: 1,
11961                            },
11962                        ));
11963                }
11964            });
11965            let mut reached = [0];
11966            barrier.read_exact(&mut reached).unwrap();
11967            assert_eq!(&reached, b"R");
11968            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
11969            let before_history = history.lock().unwrap().snapshot();
11970            barrier.write_all(b"X").unwrap();
11971            reap.join().unwrap();
11972            assert!(
11973                before_alive,
11974                "{kind:?}: death was published before recording"
11975            );
11976            assert!(before_history.entries.is_empty());
11977            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
11978            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
11979            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
11980        }
11981    }
11982
11983    #[tokio::test]
11984    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
11985        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
11986        let path = dir.join("terminals.jsonl");
11987        std::fs::create_dir(&path).unwrap();
11988        let supervisor = Supervisor::new_for_test(
11989            Arc::new(Registry::default()),
11990            RestartPolicy::new(0, Duration::ZERO),
11991        )
11992        .with_terminal_journal(path, "health-budget-journal-failure".into());
11993        let runtime = supervisor.runtime_config();
11994        let spec = ModuleSpec {
11995            program: fake_aft_stub_path(),
11996            protocol: ModuleProtocol::None,
11997            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
11998            ..windowed_crash_spec("health-budget-journal-failure")
11999        };
12000        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12001        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12002        health_restart_child(
12003            &spec,
12004            &runtime,
12005            &supervisor.registry,
12006            &supervisor.process_liveness,
12007            &snapshot,
12008            &mut child,
12009            SupervisorHealthStatus::Failing,
12010            None,
12011            1,
12012        )
12013        .await
12014        .unwrap();
12015        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12016        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12017        let history = runtime
12018            .terminal_ring
12019            .lock()
12020            .unwrap()
12021            .durable_history(&spec.module_id);
12022        assert!(history.journal_write_failures > 0);
12023        assert_eq!(history.entries.len(), 1);
12024        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12025    }
12026
12027    #[test]
12028    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12029        let supervisor = Supervisor::new_for_test(
12030            Arc::new(Registry::default()),
12031            RestartPolicy::new(0, Duration::ZERO),
12032        );
12033        let runtime = supervisor.runtime_config();
12034        let spec = ModuleSpec {
12035            program: fake_aft_stub_path(),
12036            protocol: ModuleProtocol::None,
12037            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12038            ..windowed_crash_spec("health-budget-recording-panic")
12039        };
12040        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12041        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12042            tokio::runtime::Builder::new_current_thread()
12043                .enable_all()
12044                .build()
12045                .unwrap()
12046                .block_on(async {
12047                    let mut child =
12048                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12049                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12050                    health_restart_child(
12051                        &spec,
12052                        &runtime,
12053                        &supervisor.registry,
12054                        &supervisor.process_liveness,
12055                        &snapshot,
12056                        &mut child,
12057                        SupervisorHealthStatus::Failing,
12058                        None,
12059                        1,
12060                    )
12061                    .await
12062                    .unwrap();
12063                });
12064        }));
12065        let panic = result.expect_err("recording must still unwind");
12066        assert_eq!(
12067            panic.downcast_ref::<String>().map(String::as_str),
12068            Some("spawn event sequence exhausted")
12069        );
12070        let state = lock_snapshot(&snapshot).unwrap();
12071        assert_eq!(state.state, ModuleState::Failed);
12072        assert!(!state.process_alive);
12073        assert!(state.last_exit.is_some());
12074    }
12075
12076    #[tokio::test]
12077    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12078        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12079        let path = dir.join("terminals.jsonl");
12080        std::fs::create_dir(&path).unwrap();
12081        let supervisor = Supervisor::new_for_test(
12082            Arc::new(Registry::default()),
12083            RestartPolicy::new(0, Duration::ZERO),
12084        )
12085        .with_terminal_journal(path, "budget-journal-failure".into());
12086        let runtime = supervisor.runtime_config();
12087        let spec = windowed_crash_spec("budget-journal-failure");
12088        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12089        assert!(matches!(
12090            on_child_exit(
12091                &spec,
12092                runtime.restart_policy,
12093                &supervisor.registry,
12094                &snapshot,
12095                &runtime.terminal_ring,
12096                &runtime.spawn_events,
12097                &runtime.child_roster,
12098                crash_exit_report(1),
12099            )
12100            .await,
12101            NextAction::Stop { .. }
12102        ));
12103        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12104        let history = runtime
12105            .terminal_ring
12106            .lock()
12107            .unwrap()
12108            .durable_history(&spec.module_id);
12109        assert!(history.journal_write_failures > 0);
12110        assert_eq!(history.entries.len(), 1);
12111        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12112    }
12113
12114    #[test]
12115    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12116        let supervisor = Supervisor::new_for_test(
12117            Arc::new(Registry::default()),
12118            RestartPolicy::new(0, Duration::ZERO),
12119        );
12120        let runtime = supervisor.runtime_config();
12121        let spec = windowed_crash_spec("budget-recording-panic");
12122        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12123        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12124        // Exhausting the event sequence makes emit_exited panic before the
12125        // terminal write, exercising publication on the recording unwind.
12126        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12127        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12128            tokio::runtime::Builder::new_current_thread()
12129                .enable_all()
12130                .build()
12131                .unwrap()
12132                .block_on(on_child_exit(
12133                    &spec,
12134                    runtime.restart_policy,
12135                    &supervisor.registry,
12136                    &snapshot,
12137                    &runtime.terminal_ring,
12138                    &runtime.spawn_events,
12139                    &runtime.child_roster,
12140                    crash_exit_report(1),
12141                ))
12142        }));
12143        let panic = result.err().expect("recording must still unwind");
12144        assert_eq!(
12145            panic.downcast_ref::<String>().map(String::as_str),
12146            Some("spawn event sequence exhausted")
12147        );
12148        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12149    }
12150
12151    /// A stop the supervisor itself requests still stops a protocol-none
12152    /// module, even though the child answers the SIGTERM with exit 0.
12153    #[cfg(unix)]
12154    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12155    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12156        for disable in [false, true] {
12157            let label = if disable {
12158                "none-requested-disable"
12159            } else {
12160                "none-requested-stop"
12161            };
12162            let dir = cortexkit_test_support::ScratchDir::new(label);
12163            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12164            let supervisor = Supervisor::new_for_test(
12165                Arc::new(Registry::default()),
12166                RestartPolicy::new(3, Duration::ZERO),
12167            );
12168            let module = supervisor.spawn(spec).unwrap();
12169            wait_for_file(&ready).await;
12170
12171            if disable {
12172                module.set_enabled(false).await.unwrap();
12173            } else {
12174                module.stop().await.unwrap();
12175            }
12176            assert!(
12177                marker.exists(),
12178                "{label}: the child must have left through its SIGTERM handler with exit 0"
12179            );
12180
12181            // Long enough for a zero-backoff respawn to have happened if the
12182            // exit had been treated as a crash.
12183            sleep(Duration::from_millis(500)).await;
12184            let status = module.status().unwrap();
12185            let expected = if disable {
12186                ModuleState::Disabled
12187            } else {
12188                ModuleState::Stopped
12189            };
12190            assert_eq!(status.state, expected, "{label}");
12191            assert_eq!(
12192                status.spawn_generation, 1,
12193                "{label}: respawned after a requested stop"
12194            );
12195            let history = module.terminal_history();
12196            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12197            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12198            assert_ne!(
12199                history.entries[0].disposition,
12200                TerminalDisposition::Restarting,
12201                "{label}"
12202            );
12203        }
12204    }
12205
12206    /// A subc-wire module that exits 0 on its own is still a stop: the
12207    /// protocol-none rule must not reach it.
12208    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12209    async fn subc_wire_clean_exit_is_still_a_stop() {
12210        let supervisor = Supervisor::new_for_test(
12211            Arc::new(Registry::default()),
12212            RestartPolicy::new(3, Duration::ZERO),
12213        );
12214        let module = supervisor
12215            .spawn(ModuleSpec {
12216                module_id: "wire-clean-exit".to_string(),
12217                program: fake_aft_stub_path(),
12218                args: Vec::new(),
12219                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12220                reserved: false,
12221                reserved_prefixes: Vec::new(),
12222                protocol: ModuleProtocol::Subc,
12223                overlap: Default::default(),
12224            })
12225            .unwrap();
12226
12227        let deadline = Instant::now() + Duration::from_secs(10);
12228        while module.terminal_history().entries.is_empty() {
12229            assert!(Instant::now() < deadline, "module never exited");
12230            sleep(Duration::from_millis(10)).await;
12231        }
12232        // Long enough for a zero-backoff respawn to have happened.
12233        sleep(Duration::from_millis(500)).await;
12234        let status = module.status().unwrap();
12235        assert_eq!(status.state, ModuleState::Stopped);
12236        assert_eq!(status.spawn_generation, 1);
12237        let history = module.terminal_history();
12238        assert_eq!(history.entries.len(), 1, "{history:?}");
12239        assert_eq!(history.entries[0].exit_code, Some(0));
12240        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12241    }
12242
12243    #[cfg(unix)]
12244    #[tokio::test]
12245    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12246        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12247        let record = dir.join("live-children.json");
12248        let supervisor = Supervisor::new_for_test(
12249            Arc::new(Registry::default()),
12250            RestartPolicy::new(0, Duration::ZERO),
12251        );
12252        let mut runtime = supervisor.runtime_config();
12253        runtime.child_roster.record_to(record.clone());
12254        let gate = Arc::new(super::ReloadExitRecordGate::default());
12255        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12256        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12257        let spec = ModuleSpec {
12258            module_id: "reload-exit-roster".into(),
12259            program: fake_aft_stub_path(),
12260            args: Vec::new(),
12261            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12262            reserved: false,
12263            reserved_prefixes: Vec::new(),
12264            protocol: ModuleProtocol::Subc,
12265            overlap: Default::default(),
12266        };
12267        let mut child = None;
12268        let reload = super::finish_reload_child(
12269            &spec,
12270            &runtime,
12271            &supervisor.registry,
12272            &supervisor.process_liveness,
12273            &snapshot,
12274            &mut child,
12275        );
12276        tokio::pin!(reload);
12277        tokio::select! {
12278            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12279            _ = gate.reached.notified() => {}
12280        }
12281        assert!(runtime
12282            .terminal_ring
12283            .lock()
12284            .unwrap()
12285            .snapshot()
12286            .entries
12287            .is_empty());
12288        assert_eq!(
12289            crate::live_children::read_record(&record).unwrap().len(),
12290            1,
12291            "shutdown must still wait for the reaped child until its terminal record exists"
12292        );
12293        runtime.child_roster.close();
12294        gate.resume.notify_one();
12295        assert!(reload.await.is_err());
12296        assert!(crate::live_children::read_record(&record)
12297            .unwrap()
12298            .is_empty());
12299        let history = runtime.terminal_ring.lock().unwrap().snapshot();
12300        assert_eq!(history.entries.len(), 1);
12301        assert_eq!(
12302            history.entries[0].disposition,
12303            TerminalDisposition::DaemonShutdown
12304        );
12305    }
12306
12307    /// Each restart-producing arm has its own state transition. Keeping their
12308    /// lifetime count assertions adjacent prevents a later new arm from silently
12309    /// spending budget without recording the historical restart.
12310    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12311    async fn every_restart_increment_path_advances_lifetime_count() {
12312        let supervisor = Supervisor::new_for_test(
12313            Arc::new(Registry::default()),
12314            RestartPolicy::new(1, Duration::ZERO),
12315        );
12316        let runtime = supervisor.runtime_config();
12317        let spec = ModuleSpec {
12318            module_id: "lifetime-increment-path".to_string(),
12319            program: PathBuf::from("/unused/lifetime-increment-path"),
12320            args: Vec::new(),
12321            env: Vec::new(),
12322            reserved: false,
12323            reserved_prefixes: Vec::new(),
12324            protocol: ModuleProtocol::Subc,
12325            overlap: Default::default(),
12326        };
12327
12328        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12329        assert!(matches!(
12330            on_child_exit(
12331                &spec,
12332                runtime.restart_policy,
12333                &supervisor.registry,
12334                &crash_snapshot,
12335                &runtime.terminal_ring,
12336                &runtime.spawn_events,
12337                &runtime.child_roster,
12338                ExitReport {
12339                    kind: ExitKind::Crash,
12340                    code: Some(1),
12341                    signal: None,
12342                    at_ms: 1,
12343                },
12344            )
12345            .await,
12346            NextAction::Restart { schedule: _ }
12347        ));
12348        let (crash_restarts, crash_lifetime) = {
12349            let state = lock_snapshot(&crash_snapshot).unwrap();
12350            (state.crash_restarts.len(), state.lifetime_restarts)
12351        };
12352        assert_eq!(crash_restarts, 1);
12353        assert_eq!(crash_lifetime, 1);
12354
12355        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12356        let mut health_child = None;
12357        assert!(matches!(
12358            health_restart_child(
12359                &spec,
12360                &runtime,
12361                &supervisor.registry,
12362                &supervisor.process_liveness,
12363                &health_snapshot,
12364                &mut health_child,
12365                SupervisorHealthStatus::Failing,
12366                None,
12367                2,
12368            )
12369            .await,
12370            Ok(())
12371        ));
12372        assert!(health_child.is_none());
12373        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12374        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12375        let (health_restarts, health_lifetime) = {
12376            let state = lock_snapshot(&health_snapshot).unwrap();
12377            (state.crash_restarts.len(), state.lifetime_restarts)
12378        };
12379        assert_eq!(health_restarts, 1);
12380        assert_eq!(health_lifetime, 1);
12381
12382        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12383        let mut reload_child = None;
12384        assert!(matches!(
12385            handle_reload_spawn_failure(
12386                &spec,
12387                &runtime,
12388                &supervisor.process_liveness,
12389                &reload_snapshot,
12390                &mut reload_child,
12391                "forced reload spawn failure".to_string(),
12392            )
12393            .await,
12394            Err(SuperviseError::ReloadFailed { .. })
12395        ));
12396        let (reload_restarts, reload_lifetime) = {
12397            let state = lock_snapshot(&reload_snapshot).unwrap();
12398            (state.crash_restarts.len(), state.lifetime_restarts)
12399        };
12400        assert_eq!(reload_restarts, 1);
12401        assert_eq!(reload_lifetime, 1);
12402    }
12403
12404    #[tokio::test]
12405    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12406        let supervisor = Supervisor::new_for_test(
12407            Arc::new(Registry::default()),
12408            RestartPolicy::new(3, Duration::ZERO),
12409        );
12410        let runtime = supervisor.runtime_config();
12411        let spec = ModuleSpec {
12412            module_id: "deliberately-severed".to_string(),
12413            program: PathBuf::from("/unused/deliberately-severed"),
12414            args: Vec::new(),
12415            env: Vec::new(),
12416            reserved: false,
12417            reserved_prefixes: Vec::new(),
12418            protocol: ModuleProtocol::Subc,
12419            overlap: Default::default(),
12420        };
12421        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12422        let process = ProcessIdentity {
12423            pid: 41,
12424            start_time: 101,
12425        };
12426        record_deliberate_severance(&snapshot, process).unwrap();
12427        let exit_report = apply_deliberate_severance_marker(
12428            &snapshot,
12429            Some(process),
12430            ExitReport {
12431                kind: ExitKind::Crash,
12432                code: Some(1),
12433                signal: None,
12434                at_ms: 1,
12435            },
12436        );
12437        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12438
12439        assert!(matches!(
12440            on_child_exit(
12441                &spec,
12442                runtime.restart_policy,
12443                &supervisor.registry,
12444                &snapshot,
12445                &runtime.terminal_ring,
12446                &runtime.spawn_events,
12447                &runtime.child_roster,
12448                exit_report,
12449            )
12450            .await,
12451            NextAction::Restart { schedule: _ }
12452        ));
12453        let state = lock_snapshot(&snapshot).unwrap();
12454        assert_eq!(state.lifetime_restarts, 1);
12455        assert_eq!(state.crash_restarts.len(), 0);
12456    }
12457
12458    #[tokio::test]
12459    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
12460        let supervisor = Supervisor::new_for_test(
12461            Arc::new(Registry::default()),
12462            RestartPolicy::new(3, Duration::ZERO),
12463        );
12464        let runtime = supervisor.runtime_config();
12465        let spec = ModuleSpec {
12466            module_id: "genuine-crash".to_string(),
12467            program: PathBuf::from("/unused/genuine-crash"),
12468            args: Vec::new(),
12469            env: Vec::new(),
12470            reserved: false,
12471            reserved_prefixes: Vec::new(),
12472            protocol: ModuleProtocol::Subc,
12473            overlap: Default::default(),
12474        };
12475        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12476
12477        assert!(matches!(
12478            on_child_exit(
12479                &spec,
12480                runtime.restart_policy,
12481                &supervisor.registry,
12482                &snapshot,
12483                &runtime.terminal_ring,
12484                &runtime.spawn_events,
12485                &runtime.child_roster,
12486                ExitReport {
12487                    kind: ExitKind::Crash,
12488                    code: Some(1),
12489                    signal: None,
12490                    at_ms: 1,
12491                },
12492            )
12493            .await,
12494            NextAction::Restart { schedule: _ }
12495        ));
12496        let state = lock_snapshot(&snapshot).unwrap();
12497        assert_eq!(state.lifetime_restarts, 1);
12498        assert_eq!(state.crash_restarts.len(), 1);
12499    }
12500
12501    fn crash_exit_report(at_ms: u64) -> ExitReport {
12502        ExitReport {
12503            kind: ExitKind::Crash,
12504            code: Some(1),
12505            signal: None,
12506            at_ms,
12507        }
12508    }
12509
12510    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
12511        ModuleSpec {
12512            module_id: module_id.to_string(),
12513            program: PathBuf::from("/unused").join(module_id),
12514            args: Vec::new(),
12515            env: Vec::new(),
12516            reserved: false,
12517            reserved_prefixes: Vec::new(),
12518            protocol: ModuleProtocol::Subc,
12519            overlap: Default::default(),
12520        }
12521    }
12522
12523    /// A real crash loop still stops. Three crashes with nothing aging out spend
12524    /// a budget of two and the third respawn is refused, and both surfaces an
12525    /// operator has -- the log line and the retained terminal record -- name the
12526    /// window rather than only the cap, because `max_restarts=2` alone is what
12527    /// this budget used to mean.
12528    #[tokio::test]
12529    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
12530        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
12531        let supervisor = Supervisor::new_for_test(
12532            Arc::new(Registry::default()),
12533            RestartPolicy::new(2, Duration::ZERO),
12534        );
12535        let runtime = supervisor.runtime_config();
12536        let spec = windowed_crash_spec("crash-loop-in-window");
12537        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12538
12539        for attempt in 1..=2 {
12540            assert!(
12541                matches!(
12542                    on_child_exit(
12543                        &spec,
12544                        runtime.restart_policy,
12545                        &supervisor.registry,
12546                        &snapshot,
12547                        &runtime.terminal_ring,
12548                        &runtime.spawn_events,
12549                        &runtime.child_roster,
12550                        crash_exit_report(attempt),
12551                    )
12552                    .await,
12553                    NextAction::Restart { schedule: _ }
12554                ),
12555                "crash {attempt} is inside the budget and must respawn"
12556            );
12557        }
12558
12559        assert!(matches!(
12560            on_child_exit(
12561                &spec,
12562                runtime.restart_policy,
12563                &supervisor.registry,
12564                &snapshot,
12565                &runtime.terminal_ring,
12566                &runtime.spawn_events,
12567                &runtime.child_roster,
12568                crash_exit_report(3),
12569            )
12570            .await,
12571            NextAction::Stop { .. }
12572        ));
12573
12574        {
12575            let state = lock_snapshot(&snapshot).unwrap();
12576            assert_eq!(state.state, ModuleState::Failed);
12577            assert_eq!(state.crash_restarts.len(), 2);
12578            assert_eq!(state.lifetime_restarts, 2);
12579        }
12580
12581        let history = runtime
12582            .terminal_ring
12583            .lock()
12584            .expect("terminal ring is not poisoned")
12585            .snapshot();
12586        let last = history
12587            .entries
12588            .last()
12589            .expect("the refused crash is retained");
12590        assert_eq!(last.disposition, TerminalDisposition::Failed);
12591        assert_eq!(
12592            last.disposition_detail.as_deref(),
12593            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
12594        );
12595
12596        let captured = crate::router::test_log::captured_logs(&logs);
12597        assert!(
12598            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
12599            "the stop must be logged with its window: {captured}"
12600        );
12601    }
12602
12603    /// The rate, stated as a test: three crashes where the first has aged past
12604    /// the window are two crashes as far as the budget is concerned, so the
12605    /// third respawn is allowed and the ring holds only the two recent ones.
12606    ///
12607    /// This is the case a lifetime counter got wrong -- and the case the daemon
12608    /// now hits routinely, since a module exits non-zero every time its
12609    /// connection to the daemon drops.
12610    #[tokio::test]
12611    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
12612        let supervisor = Supervisor::new_for_test(
12613            Arc::new(Registry::default()),
12614            RestartPolicy::new(2, Duration::ZERO),
12615        );
12616        let runtime = supervisor.runtime_config();
12617        let spec = windowed_crash_spec("crash-across-windows");
12618        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12619
12620        for attempt in 1..=2 {
12621            assert!(matches!(
12622                on_child_exit(
12623                    &spec,
12624                    runtime.restart_policy,
12625                    &supervisor.registry,
12626                    &snapshot,
12627                    &runtime.terminal_ring,
12628                    &runtime.spawn_events,
12629                    &runtime.child_roster,
12630                    crash_exit_report(attempt),
12631                )
12632                .await,
12633                NextAction::Restart { schedule: _ }
12634            ));
12635        }
12636
12637        // The oldest crash moves out of the window; nothing else about the
12638        // module changes.
12639        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12640            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
12641        })
12642        .unwrap();
12643
12644        assert!(
12645            matches!(
12646                on_child_exit(
12647                    &spec,
12648                    runtime.restart_policy,
12649                    &supervisor.registry,
12650                    &snapshot,
12651                    &runtime.terminal_ring,
12652                    &runtime.spawn_events,
12653                    &runtime.child_roster,
12654                    crash_exit_report(3),
12655                )
12656                .await,
12657                NextAction::Restart { schedule: _ }
12658            ),
12659            "a crash older than the window must not hold a budget slot"
12660        );
12661
12662        let state = lock_snapshot(&snapshot).unwrap();
12663        assert_eq!(state.state, ModuleState::Restarting);
12664        assert_eq!(
12665            state.crash_restarts.len(),
12666            2,
12667            "the aged instant is dropped and the new one takes its place"
12668        );
12669        assert_eq!(
12670            state.lifetime_restarts, 3,
12671            "the ledger counts every restart, including the ones the window forgot"
12672        );
12673    }
12674
12675    /// An operator restart hands the budget back whole, and the ledger keeps
12676    /// counting. Those are different questions -- "how close is this module to
12677    /// being stopped" and "how many times has it been replaced" -- and the
12678    /// operator action answers only the first.
12679    #[tokio::test]
12680    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
12681        let supervisor = Supervisor::new_for_test(
12682            Arc::new(Registry::default()),
12683            RestartPolicy::new(2, Duration::ZERO),
12684        );
12685        let runtime = supervisor.runtime_config();
12686        let spec = windowed_crash_spec("operator-cleared-budget");
12687        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12688
12689        for attempt in 1..=2 {
12690            assert!(matches!(
12691                on_child_exit(
12692                    &spec,
12693                    runtime.restart_policy,
12694                    &supervisor.registry,
12695                    &snapshot,
12696                    &runtime.terminal_ring,
12697                    &runtime.spawn_events,
12698                    &runtime.child_roster,
12699                    crash_exit_report(attempt),
12700                )
12701                .await,
12702                NextAction::Restart { schedule: _ }
12703            ));
12704        }
12705
12706        reset_restart_count(&snapshot, &spec.module_id).unwrap();
12707        {
12708            let state = lock_snapshot(&snapshot).unwrap();
12709            assert!(
12710                state.crash_restarts.is_empty(),
12711                "an operator restart returns the full budget"
12712            );
12713            assert_eq!(
12714                state.lifetime_restarts, 2,
12715                "clearing the budget must not unmake the crashes"
12716            );
12717        }
12718
12719        assert!(
12720            matches!(
12721                on_child_exit(
12722                    &spec,
12723                    runtime.restart_policy,
12724                    &supervisor.registry,
12725                    &snapshot,
12726                    &runtime.terminal_ring,
12727                    &runtime.spawn_events,
12728                    &runtime.child_roster,
12729                    crash_exit_report(3),
12730                )
12731                .await,
12732                NextAction::Restart { schedule: _ }
12733            ),
12734            "the cleared budget must be spendable again"
12735        );
12736        let state = lock_snapshot(&snapshot).unwrap();
12737        assert_eq!(state.crash_restarts.len(), 1);
12738        assert_eq!(state.lifetime_restarts, 3);
12739    }
12740
12741    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12742    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
12743        let severed = ProcessIdentity {
12744            pid: 41,
12745            start_time: 101,
12746        };
12747        let successor = ProcessIdentity {
12748            pid: 41,
12749            start_time: 202,
12750        };
12751        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12752        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
12753            state.pid = Some(successor.pid);
12754            state.process_start_time = Some(successor.start_time);
12755        })
12756        .unwrap();
12757        assert!(!module.record_deliberate_severance(severed).unwrap());
12758
12759        let exit_report = apply_deliberate_severance_marker(
12760            &module.inner.snapshot,
12761            Some(successor),
12762            ExitReport {
12763                kind: ExitKind::Crash,
12764                code: Some(1),
12765                signal: None,
12766                at_ms: 1,
12767            },
12768        );
12769
12770        assert_eq!(exit_report.kind, ExitKind::Crash);
12771    }
12772
12773    #[tokio::test]
12774    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
12775        let registry = Registry::default();
12776        let supervisor = Supervisor::new_for_test(
12777            Arc::new(Registry::default()),
12778            RestartPolicy::new(3, Duration::ZERO),
12779        );
12780        let runtime = supervisor.runtime_config();
12781        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12782        let spec = ModuleSpec {
12783            module_id: "drain-deliberate-severance".to_string(),
12784            program: fake_aft_stub_path(),
12785            args: Vec::new(),
12786            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12787            reserved: false,
12788            reserved_prefixes: Vec::new(),
12789            protocol: ModuleProtocol::Subc,
12790            overlap: Default::default(),
12791        };
12792        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
12793        let process = ProcessIdentity {
12794            pid: 41,
12795            start_time: 101,
12796        };
12797        child.process_identity = Some(process);
12798        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12799            state.pid = Some(process.pid);
12800            state.process_start_time = Some(process.start_time);
12801        })
12802        .unwrap();
12803        record_deliberate_severance(&snapshot, process).unwrap();
12804
12805        drain_child_to_state(
12806            &spec.module_id,
12807            spec.protocol,
12808            // The child exits on its own; no signal may change the exit this
12809            // test classifies.
12810            StopNotice::SentOverConnection,
12811            &registry,
12812            None,
12813            &snapshot,
12814            &runtime.terminal_ring,
12815            &runtime.spawn_events,
12816            child,
12817            Duration::from_secs(1),
12818            ModuleState::Stopped,
12819            Some(false),
12820        )
12821        .await
12822        .unwrap();
12823
12824        let state = lock_snapshot(&snapshot).unwrap();
12825        assert_eq!(
12826            state.last_exit.as_ref().map(|exit| exit.kind),
12827            Some(ExitKind::DeliberateSeverance)
12828        );
12829        assert_eq!(state.lifetime_restarts, 1);
12830        assert_eq!(state.crash_restarts.len(), 0);
12831        drop(state);
12832        let history = runtime.terminal_ring.lock().unwrap().snapshot();
12833        assert_eq!(
12834            history.entries[0].exit_kind,
12835            subc_control::TerminalExitKind::DeliberateSeverance
12836        );
12837    }
12838
12839    #[tokio::test]
12840    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
12841        let registry = Registry::default();
12842        let supervisor = Supervisor::new_for_test(
12843            Arc::new(Registry::default()),
12844            RestartPolicy::new(3, Duration::ZERO),
12845        );
12846        let runtime = supervisor.runtime_config();
12847        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12848        let spec = ModuleSpec {
12849            module_id: "ordinary-drain".to_string(),
12850            program: fake_aft_stub_path(),
12851            args: Vec::new(),
12852            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12853            reserved: false,
12854            reserved_prefixes: Vec::new(),
12855            protocol: ModuleProtocol::Subc,
12856            overlap: Default::default(),
12857        };
12858        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
12859
12860        drain_child_to_state(
12861            &spec.module_id,
12862            spec.protocol,
12863            // The child exits on its own; no signal may change the exit this
12864            // test classifies.
12865            StopNotice::SentOverConnection,
12866            &registry,
12867            None,
12868            &snapshot,
12869            &runtime.terminal_ring,
12870            &runtime.spawn_events,
12871            child,
12872            Duration::from_secs(1),
12873            ModuleState::Stopped,
12874            Some(false),
12875        )
12876        .await
12877        .unwrap();
12878
12879        let state = lock_snapshot(&snapshot).unwrap();
12880        assert_eq!(
12881            state.last_exit.as_ref().map(|exit| exit.kind),
12882            Some(ExitKind::Crash)
12883        );
12884        assert_eq!(state.lifetime_restarts, 0);
12885        assert_eq!(state.crash_restarts.len(), 0);
12886    }
12887
12888    #[test]
12889    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
12890        // The server's generic fatal-routing branch only knows that the
12891        // connection failed; it does not know that the daemon deliberately
12892        // initiated a process-killing severance. Keep this seam explicit so a
12893        // future connection error path cannot silently reintroduce the stale
12894        // exemption that mislabels a later genuine crash.
12895        assert!(!include_str!("server.rs")
12896            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
12897    }
12898
12899    /// The `route.closed` `drained` value must be the quiescence wait's own
12900    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
12901    /// measurement at all and `false` is the one honest constant. This is the exact
12902    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
12903    /// on every return path, including the one that used to return early via `?`
12904    /// with `route.closing` already sent and no `route.closed` ever following.
12905    #[test]
12906    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
12907        assert!(drained_after_quiescence_wait(&Ok(true)));
12908        assert!(!drained_after_quiescence_wait(&Ok(false)));
12909        assert!(!drained_after_quiescence_wait(&Err(
12910            SuperviseError::StatePoisoned { module_id: None }
12911        )));
12912    }
12913
12914    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
12915    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
12916    /// already reaped out-of-band) still leaves a terminal record rather than none
12917    /// at all. Triggering the real `wait()` I/O error from an integration test would
12918    /// need a genuine already-reaped-child race, which is OS-specific and not
12919    /// something this suite attempts elsewhere; this test instead verifies the
12920    /// record produced for that arm end-to-end through the real `TerminalRing`, and
12921    /// the call site itself is verified by inspection to sit in that exact arm.
12922    #[test]
12923    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
12924        let ring = Arc::new(Mutex::new(TerminalRing::new(
12925            TerminalRingConfig::default(),
12926            0,
12927        )));
12928        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
12929
12930        let snapshot = ring.lock().unwrap().snapshot();
12931        assert_eq!(snapshot.entries.len(), 1);
12932        let entry = &snapshot.entries[0];
12933        assert_eq!(entry.exit_code, None);
12934        assert_eq!(entry.exit_signal, None);
12935        assert_eq!(entry.disposition, TerminalDisposition::Failed);
12936    }
12937
12938    #[test]
12939    fn wait_error_exit_path_preserves_spawn_event_density() {
12940        let feed = super::SpawnEventFeed::default();
12941        feed.configure_incarnation("wait-error-density".to_string());
12942        feed.emit_spawned("wait-error", 41, 1);
12943        let ring = Arc::new(Mutex::new(TerminalRing::new(
12944            TerminalRingConfig::default(),
12945            0,
12946        )));
12947
12948        record_wait_error_terminal("wait-error", &ring, &feed);
12949        feed.emit_spawned("after-wait-error", 42, 2);
12950
12951        let state = feed.0.lock().unwrap();
12952        let sequences = state
12953            .events
12954            .iter()
12955            .map(|event| event.cursor.seq)
12956            .collect::<Vec<_>>();
12957        assert_eq!(sequences, vec![1, 2, 3]);
12958        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
12959        assert_eq!(state.events[1].exit_code, None);
12960        assert_eq!(state.events[1].exit_signal, None);
12961    }
12962
12963    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
12964    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
12965    /// not a clean exit it never actually observed.
12966    #[test]
12967    fn wait_error_exit_report_is_classified_as_a_crash() {
12968        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
12969    }
12970}
12971
12972#[cfg(all(test, unix))]
12973mod health_event_tests {
12974    use super::*;
12975    use cortexkit_test_support::ScratchDir;
12976    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
12977
12978    struct Actor {
12979        module: SupervisedModule,
12980        registry: Arc<Registry>,
12981        forwarding: Arc<ForwardingTable>,
12982        spec: ModuleSpec,
12983        health: HealthConfig,
12984        rx: mpsc::Receiver<crate::router::OutboundFrame>,
12985        _home: ScratchDir,
12986    }
12987
12988    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
12989        // OS exec and socket readiness are real I/O. Stay runnable while waiting
12990        // for an observed condition, rather than letting paused time auto-advance
12991        // deadlines or assuming a fixed number of yields finishes the work.
12992        let deadline = std::time::Instant::now() + Duration::from_secs(10);
12993        let tick = Instant::now();
12994        loop {
12995            if let Some(value) = observe() {
12996                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
12997                return value;
12998            }
12999            assert!(
13000                std::time::Instant::now() < deadline,
13001                "timed out waiting for {reason}"
13002            );
13003            tokio::task::yield_now().await;
13004        }
13005    }
13006
13007    impl Actor {
13008        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
13009            let home = ScratchDir::new("health-events");
13010            let registry = Arc::new(Registry::default());
13011            let forwarding = Arc::new(ForwardingTable::default());
13012            let health = HealthConfig {
13013                cadence,
13014                deadline: Duration::from_secs(3600),
13015                ..HealthConfig::default()
13016            };
13017            let supervisor =
13018                Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
13019                    .with_forwarding(forwarding.clone())
13020                    .with_health_config(health.clone());
13021            let spec = ModuleSpec {
13022                module_id: "event-health-child".into(),
13023                program: PathBuf::from("/bin/sh"),
13024                // The process stays alive but has no real bus peer. Registrations
13025                // below exercise the same registry writes as accepted HELLOs.
13026                args: vec!["-c".into(), "exec sleep 600".into()],
13027                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13028                    .into_iter()
13029                    .map(|key| (key.into(), home.path().display().to_string()))
13030                    .collect(),
13031                reserved: false,
13032                reserved_prefixes: vec![],
13033                protocol,
13034                overlap: Default::default(),
13035            };
13036            let module = supervisor.spawn(spec.clone()).unwrap();
13037            let (_, rx) = mpsc::channel(8);
13038            let actor = Self {
13039                module,
13040                registry,
13041                forwarding,
13042                spec,
13043                health,
13044                rx,
13045                _home: home,
13046            };
13047            actor
13048                .wait_for_select(|checkpoint| checkpoint.generation > 0)
13049                .await;
13050            actor
13051        }
13052
13053        fn turns(&self) -> u64 {
13054            lock_snapshot(&self.module.inner.snapshot)
13055                .unwrap()
13056                .actor_turns
13057        }
13058
13059        async fn wait_for_select(
13060            &self,
13061            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13062        ) -> ActorSelectCheckpoint {
13063            wait_for("actor select checkpoint", || {
13064                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13065                state.actor_select.filter(|checkpoint| {
13066                    checkpoint.generation == state.spawn_generation
13067                        && state.state == ModuleState::Running
13068                        && state.process_alive
13069                        && state.reported_pid().is_some()
13070                        && expected(checkpoint)
13071                })
13072            })
13073            .await
13074        }
13075
13076        fn hello(&mut self, connection: u64, health: bool) {
13077            let manifest =
13078                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13079                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13080                    .build();
13081            let connection = ConnectionId::new(connection);
13082            let (tx, rx) = mpsc::channel(8);
13083            self.rx = rx;
13084            self.forwarding
13085                .register_module_connection(
13086                    connection,
13087                    self.spec.module_id.clone(),
13088                    subc_protocol::PROTOCOL_VERSION,
13089                    Concurrency::ModuleManaged,
13090                    FrameSink::new(tx),
13091                )
13092                .unwrap();
13093            self.registry
13094                .register_with_control_ops(
13095                    manifest,
13096                    subc_protocol::PROTOCOL_VERSION,
13097                    connection,
13098                    if health {
13099                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13100                    } else {
13101                        vec![]
13102                    },
13103                )
13104                .unwrap();
13105        }
13106
13107        async fn probe(&mut self) -> crate::router::OutboundFrame {
13108            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13109                Ok(frame) => Some(frame),
13110                Err(mpsc::error::TryRecvError::Empty) => None,
13111                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13112            })
13113            .await;
13114            let body: Value = serde_json::from_slice(&frame.body).unwrap();
13115            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13116            frame
13117        }
13118
13119        fn no_probe(&mut self) {
13120            assert!(matches!(
13121                self.rx.try_recv(),
13122                Err(mpsc::error::TryRecvError::Empty)
13123            ));
13124        }
13125
13126        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13127            self.forwarding
13128                .complete_module_control_rpc(
13129                    ConnectionId::new(connection),
13130                    frame.header.corr,
13131                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13132                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13133                        status: HealthStatus::Ok,
13134                        detail: None,
13135                        metrics: None,
13136                    }),
13137                )
13138                .unwrap();
13139        }
13140    }
13141
13142    #[tokio::test(start_paused = true)]
13143    async fn no_health_children_park_for_sixty_seconds() {
13144        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13145        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13146        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13147        unadvertised.hello(1, false);
13148        // Include any one-off registration wake in the budget; receiving a
13149        // no-health HELLO must not turn a parked child into a polling child.
13150        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
13151        // a single 60 s jump would only observe one expired timer.
13152        for _ in 0..6000 {
13153            tokio::time::advance(Duration::from_millis(10)).await;
13154            tokio::task::yield_now().await;
13155        }
13156        for actor in [&none, &unregistered, &unadvertised] {
13157            assert!(
13158                actor.turns() <= 5,
13159                "no-health actor woke {} times",
13160                actor.turns()
13161            );
13162            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13163        }
13164    }
13165
13166    #[tokio::test(start_paused = true)]
13167    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13168        // With a zero cadence the first probe is due at once, so this measures
13169        // only how fast the registration notification starts health probing,
13170        // not the normal 30-33 s wait before a first probe (checked below).
13171        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13172        let tick = Instant::now();
13173        actor.hello(1, true);
13174        actor.probe().await;
13175        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13176    }
13177
13178    #[tokio::test(start_paused = true)]
13179    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13180        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13181        let hello_tick = Instant::now();
13182        actor.hello(1, true);
13183        let armed = actor
13184            .wait_for_select(|checkpoint| {
13185                checkpoint.registered_connection == Some(ConnectionId::new(1))
13186                    && checkpoint.next_probe_at.is_some()
13187            })
13188            .await;
13189        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13190        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13191        let first_deadline = hello_tick + delay;
13192        assert_eq!(armed.next_probe_at, Some(first_deadline));
13193        tokio::time::advance(Duration::from_secs(29)).await;
13194        let turns = actor.turns();
13195        actor
13196            .registry
13197            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13198            .unwrap();
13199        let updated = actor
13200            .wait_for_select(|checkpoint| checkpoint.turn > turns)
13201            .await;
13202        assert_eq!(
13203            updated.next_probe_at,
13204            Some(first_deadline),
13205            "catalog update must not reset cadence"
13206        );
13207        actor.no_probe();
13208        tokio::time::advance(first_deadline - Instant::now()).await;
13209        let frame = actor.probe().await;
13210        actor.answer(1, frame);
13211        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13212        let next_deadline = Instant::now() + next;
13213        let rearmed = actor
13214            .wait_for_select(|checkpoint| {
13215                checkpoint
13216                    .next_probe_at
13217                    .is_some_and(|deadline| deadline > first_deadline)
13218            })
13219            .await;
13220        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
13221        tokio::time::advance(next - Duration::from_nanos(1)).await;
13222        actor.no_probe();
13223        tokio::time::advance(Duration::from_nanos(1)).await;
13224        actor.probe().await;
13225    }
13226
13227    #[tokio::test(start_paused = true)]
13228    async fn disconnect_disarms_and_reconnect_rearms_health() {
13229        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13230        actor.hello(1, true);
13231        actor
13232            .wait_for_select(|checkpoint| {
13233                checkpoint.registered_connection == Some(ConnectionId::new(1))
13234                    && checkpoint.next_probe_at.is_some()
13235            })
13236            .await;
13237        let turns = actor.turns();
13238        let tick = Instant::now();
13239        actor
13240            .registry
13241            .deregister_connection(ConnectionId::new(1))
13242            .unwrap();
13243        let disarmed = actor
13244            .wait_for_select(|checkpoint| {
13245                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
13246            })
13247            .await;
13248        assert_eq!(disarmed.next_probe_at, None);
13249        assert_eq!(disarmed.wake_after, None);
13250        assert_eq!(
13251            Instant::now(),
13252            tick,
13253            "disconnect must disarm in the notification tick"
13254        );
13255        tokio::time::advance(Duration::from_secs(60)).await;
13256        actor.no_probe();
13257        let reconnect_tick = Instant::now();
13258        actor.hello(2, true);
13259        let rearmed = actor
13260            .wait_for_select(|checkpoint| {
13261                checkpoint.registered_connection == Some(ConnectionId::new(2))
13262                    && checkpoint.next_probe_at.is_some()
13263            })
13264            .await;
13265        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13266        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
13267        tokio::time::advance(delay).await;
13268        actor.probe().await;
13269    }
13270
13271    #[tokio::test(start_paused = true)]
13272    async fn reregistration_without_health_disarms_the_old_schedule() {
13273        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13274        actor.hello(1, true);
13275        actor
13276            .wait_for_select(|checkpoint| {
13277                checkpoint.registered_connection == Some(ConnectionId::new(1))
13278                    && checkpoint.next_probe_at.is_some()
13279            })
13280            .await;
13281        actor
13282            .registry
13283            .deregister_connection(ConnectionId::new(1))
13284            .unwrap();
13285        actor.hello(2, false);
13286        let disarmed = actor
13287            .wait_for_select(|checkpoint| {
13288                checkpoint.registered_connection == Some(ConnectionId::new(2))
13289            })
13290            .await;
13291        assert_eq!(disarmed.next_probe_at, None);
13292        assert_eq!(disarmed.wake_after, None);
13293        tokio::time::advance(Duration::from_secs(60)).await;
13294        actor.no_probe();
13295    }
13296
13297    #[tokio::test(start_paused = true)]
13298    async fn restart_waits_for_its_own_hello_before_rearming_health() {
13299        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13300        actor.hello(1, true);
13301        let armed = actor
13302            .wait_for_select(|checkpoint| {
13303                checkpoint.registered_connection == Some(ConnectionId::new(1))
13304                    && checkpoint.next_probe_at.is_some()
13305            })
13306            .await;
13307        // Simulate the old process's connection closing before the restart
13308        // tears the process down. The replacement process has started but has
13309        // not sent its HELLO, so it is not registered.
13310        actor
13311            .registry
13312            .deregister_connection(ConnectionId::new(1))
13313            .unwrap();
13314        actor.module.restart(Some(0)).await.unwrap();
13315        let parked = actor
13316            .wait_for_select(|checkpoint| {
13317                checkpoint.generation > armed.generation
13318                    && checkpoint.registered_connection.is_none()
13319                    && checkpoint.next_probe_at.is_none()
13320            })
13321            .await;
13322        assert_eq!(parked.wake_after, None);
13323        actor.rx.close();
13324        // Drain the stop notice the restart queued for the old process's connection,
13325        // so it isn't mistaken for traffic to the replacement.
13326        while actor.rx.try_recv().is_ok() {}
13327        let turns = actor.turns();
13328        tokio::time::advance(Duration::from_secs(60)).await;
13329        let after = actor.turns();
13330        eprintln!(
13331            "restart park turns: before={turns} after={after} delta={}",
13332            after - turns
13333        );
13334        assert_eq!(after, turns, "replacement must park before HELLO");
13335        let hello_tick = Instant::now();
13336        actor.hello(2, true);
13337        let rearmed = actor
13338            .wait_for_select(|checkpoint| {
13339                checkpoint.registered_connection == Some(ConnectionId::new(2))
13340                    && checkpoint.next_probe_at.is_some()
13341            })
13342            .await;
13343        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13344        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
13345        tokio::time::advance(delay).await;
13346        actor.probe().await;
13347    }
13348
13349    #[tokio::test(start_paused = true)]
13350    async fn rescan_adds_and_removes_http_health_without_polling() {
13351        use tokio::{
13352            io::{AsyncReadExt, AsyncWriteExt},
13353            net::TcpListener,
13354        };
13355        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13356        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
13357        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
13358        // One millisecond separates successive probes without a polling delay.
13359        actor.health.cadence = Duration::from_millis(1);
13360        let (started_tx, mut started_rx) = mpsc::channel(8);
13361        let peer = tokio::spawn(async move {
13362            loop {
13363                let (mut socket, _) = listener.accept().await.unwrap();
13364                let mut request = [0; 1024];
13365                assert!(socket.read(&mut request).await.unwrap() > 0);
13366                started_tx.send(()).await.unwrap();
13367                socket
13368                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
13369                    .await
13370                    .unwrap();
13371            }
13372        });
13373        let tick = Instant::now();
13374        let turns = actor.turns();
13375        actor
13376            .module
13377            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13378            .await
13379            .unwrap();
13380        let armed = actor
13381            .wait_for_select(|checkpoint| {
13382                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
13383            })
13384            .await;
13385        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
13386        assert_eq!(Instant::now(), tick);
13387        tokio::time::advance(Duration::from_millis(1)).await;
13388        let mut started = false;
13389        wait_for("successful HTTP probe", || {
13390            started |= started_rx.try_recv().is_ok();
13391            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
13392                .then_some(())
13393        })
13394        .await;
13395        assert!(started, "added HTTP check must start at its first deadline");
13396        assert_eq!(
13397            actor.module.status().unwrap().health.status,
13398            SupervisorHealthStatus::Ok
13399        );
13400        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
13401        actor.health.http = None;
13402        let turns = actor.turns();
13403        actor
13404            .module
13405            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13406            .await
13407            .unwrap();
13408        let parked = actor
13409            .wait_for_select(|checkpoint| {
13410                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
13411            })
13412            .await;
13413        assert_eq!(parked.wake_after, None);
13414        let turns = actor.turns();
13415        tokio::time::advance(Duration::from_secs(60)).await;
13416        assert_eq!(
13417            actor.turns(),
13418            turns,
13419            "removed HTTP check must park the actor"
13420        );
13421        assert!(
13422            started_rx.try_recv().is_err(),
13423            "removed HTTP check must stay disarmed"
13424        );
13425        assert_eq!(
13426            actor.module.status().unwrap().health,
13427            ModuleHealthStatus::default()
13428        );
13429        peer.abort();
13430    }
13431}
13432
13433#[cfg(test)]
13434mod drain_busy_reprobe_tests {
13435    //! The drain's busy-gauge wait, driven against an in-process module
13436    //! connection whose health answers the test scripts. Time is paused, so
13437    //! every elapsed duration below is exact virtual time.
13438    use super::*;
13439    use std::sync::Mutex as StdMutex;
13440    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13441
13442    const MODULE_ID: &str = "drain-reprobe-module";
13443    const CONNECTION: u64 = 7;
13444    const GAUGE: &str = "runs_in_flight";
13445    /// The production default for both the health cadence and the drain
13446    /// ceiling, which is the pairing that used to cost a full ceiling.
13447    const CADENCE: Duration = Duration::from_secs(30);
13448    const CEILING: Duration = Duration::from_secs(30);
13449
13450    /// How the scripted module answers each `health.check`.
13451    #[derive(Clone, Copy)]
13452    enum Answer {
13453        /// Report the gauge busy until this instant, quiet from then on, after
13454        /// taking `latency` to answer.
13455        BusyUntil { until: Instant, latency: Duration },
13456        /// Never answer.
13457        Silent,
13458    }
13459
13460    struct Drain {
13461        forwarding: Arc<ForwardingTable>,
13462        runtime: SupervisorRuntimeConfig,
13463        endpoint: crate::ModuleEndpointId,
13464        /// When each probe reached the module.
13465        probes: Arc<StdMutex<Vec<Instant>>>,
13466        responder: tokio::task::JoinHandle<()>,
13467    }
13468
13469    impl Drain {
13470        fn start(answer: Answer) -> Self {
13471            let registry = Arc::new(Registry::default());
13472            let forwarding = Arc::new(ForwardingTable::default());
13473            let health = HealthConfig {
13474                cadence: CADENCE,
13475                ..HealthConfig::default()
13476            };
13477            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
13478                .with_forwarding(forwarding.clone())
13479                .with_health_config(health)
13480                .runtime_config();
13481            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
13482            let endpoint = forwarding
13483                .register_module_connection(
13484                    ConnectionId::new(CONNECTION),
13485                    MODULE_ID.to_string(),
13486                    subc_protocol::PROTOCOL_VERSION,
13487                    Concurrency::ModuleManaged,
13488                    FrameSink::new(tx),
13489                )
13490                .unwrap();
13491            let probes = Arc::new(StdMutex::new(Vec::new()));
13492            let responder = tokio::spawn({
13493                let forwarding = forwarding.clone();
13494                let probes = probes.clone();
13495                async move {
13496                    while let Some(frame) = rx.recv().await {
13497                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
13498                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13499                        probes.lock().unwrap().push(Instant::now());
13500                        let Answer::BusyUntil { until, latency } = answer else {
13501                            continue;
13502                        };
13503                        sleep(latency).await;
13504                        let busy = u64::from(Instant::now() < until);
13505                        // A probe abandoned at its deadline has no RPC left to
13506                        // complete, so a late answer may be refused.
13507                        let _ = forwarding.complete_module_control_rpc(
13508                            ConnectionId::new(CONNECTION),
13509                            frame.header.corr,
13510                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13511                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13512                                status: HealthStatus::Ok,
13513                                detail: None,
13514                                metrics: Some(serde_json::json!({ GAUGE: busy })),
13515                            }),
13516                        );
13517                    }
13518                }
13519            });
13520            Self {
13521                forwarding,
13522                runtime,
13523                endpoint,
13524                probes,
13525                responder,
13526            }
13527        }
13528
13529        /// Run one by-id drain wait with `ceiling`, returning its verdict and
13530        /// how long it took.
13531        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
13532            let started = Instant::now();
13533            let drained = wait_for_forwarding_quiescence(
13534                &self.forwarding,
13535                MODULE_ID,
13536                &self.runtime,
13537                self.endpoint,
13538                started + ceiling,
13539                &[GAUGE.to_string()],
13540                DrainScope::Active,
13541            )
13542            .await
13543            .unwrap();
13544            (drained, started.elapsed())
13545        }
13546
13547        fn probes(&self) -> Vec<Instant> {
13548            self.probes.lock().unwrap().clone()
13549        }
13550    }
13551
13552    impl Drop for Drain {
13553        fn drop(&mut self) {
13554            self.responder.abort();
13555        }
13556    }
13557
13558    #[tokio::test(start_paused = true)]
13559    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
13560        let drain = Drain::start(Answer::BusyUntil {
13561            until: Instant::now() + Duration::from_secs(1),
13562            latency: Duration::ZERO,
13563        });
13564        let (drained, elapsed) = drain.run(CEILING).await;
13565        assert!(
13566            drained,
13567            "gauges that read zero during the drain must let it finish drained, \
13568             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
13569        );
13570        assert!(
13571            elapsed < Duration::from_secs(3),
13572            "drain took {elapsed:?} after the gauges went quiet at 1s"
13573        );
13574        let probes = drain.probes();
13575        assert!(
13576            probes.len() >= 2,
13577            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
13578        );
13579    }
13580
13581    #[tokio::test(start_paused = true)]
13582    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
13583        let started = Instant::now();
13584        let drain = Drain::start(Answer::BusyUntil {
13585            until: started + Duration::from_secs(3600),
13586            latency: Duration::ZERO,
13587        });
13588        let (drained, elapsed) = drain.run(CEILING).await;
13589        assert!(!drained, "a module busy throughout must not drain");
13590        assert!(
13591            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
13592            "a busy module is held until the ceiling and no longer: {elapsed:?}"
13593        );
13594        // Re-probing is paced by the interval, not a busy loop: at most one
13595        // probe per interval across the ceiling, plus the first at time zero.
13596        let probes = drain.probes();
13597        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
13598        assert!(
13599            probes.len() as u128 <= max_probes,
13600            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
13601            probes.len()
13602        );
13603        for pair in probes.windows(2) {
13604            assert!(
13605                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
13606                "consecutive probes {:?} apart",
13607                pair[1] - pair[0]
13608            );
13609        }
13610    }
13611
13612    #[tokio::test(start_paused = true)]
13613    async fn a_slow_probe_is_never_overlapped_by_the_next() {
13614        // Each answer takes longer than the re-probe interval; the next probe
13615        // must wait for the previous answer and then the interval.
13616        let latency = Duration::from_secs(1);
13617        let drain = Drain::start(Answer::BusyUntil {
13618            until: Instant::now() + Duration::from_secs(3600),
13619            latency,
13620        });
13621        let (drained, _) = drain.run(Duration::from_secs(5)).await;
13622        assert!(!drained);
13623        let probes = drain.probes();
13624        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
13625        for pair in probes.windows(2) {
13626            assert!(
13627                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
13628                "a probe started {:?} after the previous one, before it could finish",
13629                pair[1] - pair[0]
13630            );
13631        }
13632    }
13633
13634    #[tokio::test(start_paused = true)]
13635    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
13636        let drain = Drain::start(Answer::Silent);
13637        let ceiling = Duration::from_secs(2);
13638        let (drained, elapsed) = drain.run(ceiling).await;
13639        assert!(!drained, "an unanswered probe counts as busy");
13640        assert!(
13641            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
13642            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
13643        );
13644        assert_eq!(
13645            drain.probes().len(),
13646            1,
13647            "the unanswered probe holds the only slot until the ceiling"
13648        );
13649    }
13650}
13651
13652#[cfg(test)]
13653mod health_evidence_tests {
13654    use super::{HealthProbeError, HealthProbeEvidence};
13655    use std::collections::HashSet;
13656
13657    /// The evidential asymmetry, asserted rather than described.
13658    ///
13659    /// Exactly ONE observation is proof a module cannot serve, and the one that
13660    /// fires under CPU starvation is not it. Before the split, all fifteen
13661    /// construction sites collapsed into a single String, so a timeout carried the
13662    /// same weight as a dead lane -- which is how a healthy module was restarted
13663    /// three times in one day.
13664    #[test]
13665    fn only_a_dead_lane_is_proof_of_death() {
13666        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
13667        // Three non-proof classes, each for a different reason: silence is
13668        // consistent with health, a bad answer proves the module ALIVE, and a
13669        // daemon-side fault never reached the module at all.
13670        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
13671        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
13672        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
13673    }
13674
13675    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
13676    ///
13677    /// A shared label renders two different observations identically in the line an
13678    /// operator reads after an unexplained restart -- the exact confusion this
13679    /// change removes.
13680    #[test]
13681    fn every_evidence_class_has_a_distinct_label() {
13682        let labels = [
13683            HealthProbeError::lane_dead("").label(),
13684            HealthProbeError::no_answer("").label(),
13685            HealthProbeError::bad_answer("").label(),
13686            HealthProbeError::misconfigured("").label(),
13687        ];
13688        let unique: HashSet<_> = labels.iter().collect();
13689        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
13690    }
13691
13692    /// The class is additional information, not a replacement.
13693    ///
13694    /// An operator needs both "this was silence" and the specific text saying how
13695    /// long we waited; a classification that swallowed the message would trade one
13696    /// missing distinction for another.
13697    #[test]
13698    fn classification_preserves_the_original_message() {
13699        let err = HealthProbeError::no_answer("module did not answer within 5s");
13700        assert_eq!(err.to_string(), "module did not answer within 5s");
13701        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13702    }
13703}
13704
13705#[cfg(test)]
13706mod health_tombstone_tests {
13707    use std::{path::PathBuf, sync::Arc, time::Duration};
13708
13709    use subc_protocol::{
13710        manifest::Concurrency,
13711        session::{HealthStatus, ModuleControlResponse},
13712    };
13713    use tokio::sync::mpsc;
13714
13715    use super::{
13716        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
13717        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
13718    };
13719    use crate::{
13720        control::ControlHandler,
13721        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
13722        registry::{ConnectionId, Registry},
13723        router::FrameSink,
13724    };
13725
13726    struct ProbeHarness {
13727        spec: ModuleSpec,
13728        runtime: SupervisorRuntimeConfig,
13729        forwarding: Arc<ForwardingTable>,
13730        module_connection: ConnectionId,
13731        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13732        handler: ControlHandler,
13733        module: super::SupervisedModule,
13734    }
13735
13736    fn probe_harness() -> ProbeHarness {
13737        let registry = Arc::new(Registry::default());
13738        let forwarding = Arc::new(ForwardingTable::default());
13739        let supervisor_handle = super::SupervisorHandle::new();
13740        let health = HealthConfig {
13741            http: None,
13742            cadence: Duration::from_secs(30),
13743            deadline: Duration::from_secs(5),
13744            failure_threshold: 3,
13745            on_degraded: HealthAction::Report,
13746            on_failing: HealthAction::Report,
13747            critical: false,
13748        };
13749        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
13750            .with_forwarding(Arc::clone(&forwarding))
13751            .with_handle(supervisor_handle.clone())
13752            .with_health_config(health);
13753        let spec = ModuleSpec {
13754            module_id: "late-health-module".to_string(),
13755            program: PathBuf::from("disabled-module"),
13756            args: Vec::new(),
13757            env: Vec::new(),
13758            reserved: false,
13759            reserved_prefixes: Vec::new(),
13760            protocol: ModuleProtocol::Subc,
13761            overlap: Default::default(),
13762        };
13763        let module = supervisor
13764            .supervise_configured(spec.clone(), false)
13765            .unwrap();
13766        let runtime = supervisor.runtime_config();
13767        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
13768            .with_supervisor(supervisor_handle);
13769        let module_connection = ConnectionId::new(700);
13770        let (module_tx, module_rx) = mpsc::channel(8);
13771        forwarding
13772            .register_module_connection(
13773                module_connection,
13774                spec.module_id.clone(),
13775                subc_protocol::PROTOCOL_VERSION,
13776                Concurrency::ModuleManaged,
13777                FrameSink::new(module_tx),
13778            )
13779            .unwrap();
13780
13781        ProbeHarness {
13782            spec,
13783            runtime,
13784            forwarding,
13785            module_connection,
13786            module_rx,
13787            handler,
13788            module,
13789        }
13790    }
13791
13792    async fn finish_after(
13793        harness: &mut ProbeHarness,
13794        stall: Duration,
13795    ) -> ModuleControlRpcCompletion {
13796        assert!(stall > harness.runtime.health.deadline);
13797        let deadline = harness.runtime.health.deadline;
13798        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
13799        let answer = async {
13800            let frame = harness.module_rx.recv().await.expect("health.check frame");
13801            tokio::time::advance(deadline).await;
13802            tokio::task::yield_now().await;
13803            tokio::time::advance(stall - deadline).await;
13804            harness
13805                .forwarding
13806                .complete_module_control_rpc(
13807                    harness.module_connection,
13808                    frame.header.corr,
13809                    Some("health.check"),
13810                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13811                        status: HealthStatus::Ok,
13812                        detail: None,
13813                        metrics: None,
13814                    }),
13815                )
13816                .unwrap()
13817        };
13818        let (probe_result, completion) = tokio::join!(probe, answer);
13819        let err = probe_result.expect_err("probe must miss its deadline");
13820        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13821        completion
13822    }
13823
13824    async fn time_out_without_answer(harness: &mut ProbeHarness) {
13825        let deadline = harness.runtime.health.deadline;
13826        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
13827        let exhaust_deadline = async {
13828            let _frame = harness.module_rx.recv().await.expect("health.check frame");
13829            tokio::time::advance(deadline).await;
13830            tokio::task::yield_now().await;
13831        };
13832        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
13833        let err = probe_result.expect_err("probe must miss its deadline");
13834        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13835    }
13836
13837    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
13838        let registry = Arc::clone(&harness.module.inner.registry);
13839        let snapshot = Arc::clone(&harness.module.inner.snapshot);
13840        let process_liveness = super::SupervisorProcessLiveness::default();
13841        let mut child = None;
13842        let cycle = super::run_health_probe_cycle(
13843            &harness.spec,
13844            &harness.runtime,
13845            &registry,
13846            &process_liveness,
13847            &snapshot,
13848            &mut child,
13849        );
13850        let peer = async {
13851            let frame = harness.module_rx.recv().await.expect("health.check frame");
13852            if answer {
13853                harness
13854                    .forwarding
13855                    .complete_module_control_rpc(
13856                        harness.module_connection,
13857                        frame.header.corr,
13858                        Some("health.check"),
13859                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13860                            status: HealthStatus::Ok,
13861                            detail: None,
13862                            metrics: Some(serde_json::json!({"ready": true})),
13863                        }),
13864                    )
13865                    .unwrap();
13866            } else {
13867                tokio::time::advance(harness.runtime.health.deadline).await;
13868                tokio::task::yield_now().await;
13869            }
13870        };
13871        tokio::join!(cycle, peer);
13872    }
13873
13874    #[tokio::test(start_paused = true)]
13875    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
13876        let mut harness = probe_harness();
13877        // Drive the probe cycle directly with an in-memory wire peer. Stop the
13878        // disabled module's monitor so only this test owns lifecycle transitions;
13879        // no OS process is launched, and a restart is observed at scheduling.
13880        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
13881        monitor.abort();
13882        let _ = monitor.await;
13883        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
13884            state.enabled = true;
13885            state.state = super::ModuleState::Running;
13886            state.process_alive = true;
13887        })
13888        .unwrap();
13889
13890        run_probe_cycle(&mut harness, true).await;
13891        assert_eq!(
13892            harness.module.status().unwrap().health.status,
13893            super::SupervisorHealthStatus::Ok
13894        );
13895
13896        for failures in 1..harness.runtime.health.failure_threshold {
13897            run_probe_cycle(&mut harness, false).await;
13898            let status = harness.module.status().unwrap();
13899            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
13900            assert_eq!(status.health.consecutive_failures, failures);
13901            assert!(status.health.last_probe_ms.is_some());
13902            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
13903            assert!(status.health.metrics.is_none());
13904            assert_eq!(status.state, super::ModuleState::Running);
13905            assert!(status.process_alive);
13906            assert_eq!(status.restart_count, 0);
13907            assert_eq!(status.lifetime_restarts, 0);
13908            assert!(status.health.last_action.is_none());
13909            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
13910        }
13911
13912        run_probe_cycle(&mut harness, true).await;
13913        let recovered = harness.module.status().unwrap();
13914        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
13915        assert_eq!(recovered.health.consecutive_failures, 0);
13916        assert!(recovered.health.detail.is_none());
13917        assert_eq!(
13918            recovered.health.metrics,
13919            Some(serde_json::json!({"ready": true}))
13920        );
13921        assert_eq!(recovered.lifetime_restarts, 0);
13922
13923        for failures in 1..=harness.runtime.health.failure_threshold {
13924            run_probe_cycle(&mut harness, false).await;
13925            let status = harness.module.status().unwrap();
13926            assert_eq!(status.health.consecutive_failures, failures);
13927            if failures < harness.runtime.health.failure_threshold {
13928                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
13929                assert_eq!(status.state, super::ModuleState::Running);
13930                assert_eq!(status.lifetime_restarts, 0);
13931                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
13932            } else {
13933                assert_eq!(
13934                    status.health.status,
13935                    super::SupervisorHealthStatus::Unresponsive
13936                );
13937                assert_eq!(status.state, super::ModuleState::Restarting);
13938                assert_eq!(status.restart_count, 1);
13939                assert_eq!(status.lifetime_restarts, 1);
13940                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
13941                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
13942            }
13943        }
13944    }
13945
13946    #[tokio::test(start_paused = true)]
13947    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
13948        let mut harness = probe_harness();
13949
13950        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
13951        let first_latency = match &first {
13952            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
13953            other => panic!("late answer was not retained: {other:?}"),
13954        };
13955        assert!(harness.handler.observe_module_control_completion(first));
13956
13957        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
13958        let second_latency = match &second {
13959            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
13960            other => panic!("late answer was not retained: {other:?}"),
13961        };
13962        assert!(harness.handler.observe_module_control_completion(second));
13963
13964        assert_eq!(first_latency, Duration::from_secs(8));
13965        assert_eq!(
13966            second_latency - first_latency,
13967            Duration::from_secs(3),
13968            "latency must grow linearly with the additional stall"
13969        );
13970        let health = harness.module.status().unwrap().health;
13971        assert_eq!(health.late_answer_count, 2);
13972        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
13973    }
13974
13975    /// A module that answers every probe late must never march to the kill
13976    /// threshold: the late answer proves it is alive, so it must clear the miss
13977    /// streak the timeout recorded. Without the reset, a CPU-starved module
13978    /// that serves every probe seconds past the deadline accumulates
13979    /// `consecutive_failures` to the threshold and is killed — the exact
13980    /// sequence from the 2026-08-14 aft disable, where the daemon logged
13981    /// "proves the module is alive" five times while counting five misses.
13982    #[tokio::test(start_paused = true)]
13983    async fn late_answer_clears_the_consecutive_failure_streak() {
13984        let mut harness = probe_harness();
13985
13986        // Timeout recorded first: the probe path saw no answer in time.
13987        time_out_without_answer(&mut harness).await;
13988        harness
13989            .module
13990            .record_health_probe_failure_for_test("[no-answer] test miss")
13991            .unwrap();
13992        assert_eq!(
13993            harness.module.status().unwrap().health.consecutive_failures,
13994            1,
13995            "precondition: the miss must be on the streak before the late answer"
13996        );
13997
13998        // The stalled reply then lands: proof of life.
13999        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
14000        assert!(matches!(
14001            late,
14002            ModuleControlRpcCompletion::LateHealthAnswer { .. }
14003        ));
14004        assert!(harness.handler.observe_module_control_completion(late));
14005
14006        let health = harness.module.status().unwrap().health;
14007        assert_eq!(
14008            health.consecutive_failures, 0,
14009            "a late answer is an answer: the streak must reset"
14010        );
14011        assert_eq!(health.late_answer_count, 1);
14012    }
14013
14014    #[tokio::test(start_paused = true)]
14015    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
14016        let mut harness = probe_harness();
14017
14018        for _ in 0..20 {
14019            time_out_without_answer(&mut harness).await;
14020            assert_eq!(
14021                harness.forwarding.health_probe_tombstone_count().unwrap(),
14022                1
14023            );
14024        }
14025    }
14026}
14027
14028#[cfg(test)]
14029mod child_env_tests {
14030    #[cfg(windows)]
14031    use super::{
14032        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
14033    };
14034    #[cfg(windows)]
14035    use std::sync::{Arc, OnceLock};
14036    #[cfg(windows)]
14037    #[test]
14038    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
14039        let spec = spec(Vec::new());
14040        let mut command = Command::new("unused");
14041        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
14042            .unwrap()
14043            .unwrap();
14044        let name = command
14045            .as_std()
14046            .get_envs()
14047            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
14048            .and_then(|(_, value)| value)
14049            .unwrap();
14050        assert_eq!(name, OsStr::new(handoff.name()));
14051        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
14052        assert!(command.as_std().get_envs().any(|(key, value)| key
14053            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14054            && value == Some(OsStr::new("env"))));
14055        assert!(command
14056            .as_std()
14057            .get_envs()
14058            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
14059    }
14060
14061    #[cfg(windows)]
14062    #[test]
14063    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
14064        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
14065        snapshot.process_alive = true;
14066        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
14067        assert_eq!(
14068            observed_launch_nonce_source(&snapshot).as_deref(),
14069            Some("env")
14070        );
14071        let consumed = Arc::new(OnceLock::new());
14072        snapshot.pipe_consumed = Some(consumed.clone());
14073        assert_eq!(
14074            observed_launch_nonce_source(&snapshot).as_deref(),
14075            Some("env")
14076        );
14077        consumed.set(()).unwrap();
14078        assert_eq!(
14079            observed_launch_nonce_source(&snapshot).as_deref(),
14080            Some("pipe")
14081        );
14082        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
14083        assert_eq!(
14084            observed_launch_nonce_source(&snapshot).as_deref(),
14085            Some("env")
14086        );
14087        clear_current_process_facts(&mut snapshot);
14088        assert_eq!(observed_launch_nonce_source(&snapshot), None);
14089    }
14090    use super::{
14091        apply_child_env, apply_spawn_role, apply_wire_spawn_args, ModuleProtocol, ModuleSpec,
14092        SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG, SUBC_LAUNCH_NONCE_ENV,
14093        SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
14094    };
14095    use std::{ffi::OsStr, path::PathBuf};
14096    use tokio::process::Command;
14097
14098    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
14099        ModuleSpec {
14100            module_id: "env-plan".to_string(),
14101            program: PathBuf::from("/nonexistent"),
14102            args: Vec::new(),
14103            env,
14104            reserved: false,
14105            reserved_prefixes: Vec::new(),
14106            protocol: ModuleProtocol::Subc,
14107            overlap: Default::default(),
14108        }
14109    }
14110
14111    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
14112    /// one still gets its own.
14113    ///
14114    /// This is the narrow goal `env_clear()` was reached for, and the reason the
14115    /// fix is `env_remove` rather than deleting the line: an operator's ambient
14116    /// filter silently becoming an unconfigured module's log level is a real
14117    /// defect, just a much smaller one than clearing the environment.
14118    ///
14119    /// Asserted on the command plan rather than a spawned child because proving
14120    /// the ABSENCE of an inherited variable needs the parent's environment
14121    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
14122    /// removal as `(key, None)`, which is exactly the distinction wanted: not
14123    /// "absent because nobody set it" but "explicitly unset for the child".
14124    #[test]
14125    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
14126        let mut command = Command::new("/nonexistent");
14127        apply_child_env(&mut command, &spec(Vec::new()));
14128        let removed = command
14129            .as_std()
14130            .get_envs()
14131            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
14132        assert!(
14133            removed,
14134            "ambient CK_LOG must be explicitly removed for an unconfigured module"
14135        );
14136
14137        let mut configured = Command::new("/nonexistent");
14138        apply_child_env(
14139            &mut configured,
14140            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
14141        );
14142        let effective = configured
14143            .as_std()
14144            .get_envs()
14145            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
14146            .last()
14147            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14148        assert_eq!(
14149            effective,
14150            Some(Some("debug".to_string())),
14151            "a module's configured CK_LOG must survive the ambient removal"
14152        );
14153    }
14154
14155    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
14156    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
14157    /// the same reason as the CK_LOG test above.
14158    ///
14159    /// The argument is the load-bearing half: a stock binary exits on an
14160    /// unknown flag before it listens, so with `--subc` appended the mode
14161    /// could not supervise the one process it exists for. Found by the first
14162    /// conformance run (nats-server: `flag provided but not defined: -subc`).
14163    #[test]
14164    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
14165        let connection_file = std::path::Path::new("/run/subc-connection.json");
14166        let handle = SupervisorHandle::new();
14167
14168        let mut none_spec = spec(Vec::new());
14169        none_spec.protocol = ModuleProtocol::None;
14170        let mut none = Command::new("/nonexistent");
14171        let none_handoff =
14172            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
14173                .expect("protocol-none spawn args apply");
14174        assert!(
14175            none_handoff.is_none(),
14176            "protocol:none spawn must not receive a nonce descriptor"
14177        );
14178        #[cfg(windows)]
14179        assert!(!none.as_std().get_envs().any(|(key, value)| key
14180            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14181            && value.is_some()));
14182        assert!(
14183            !none.as_std().get_envs().any(|(key, value)| key
14184                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
14185                && value.is_some()),
14186            "protocol:none spawn must not name a nonce descriptor"
14187        );
14188        let none_args: Vec<String> = none
14189            .as_std()
14190            .get_args()
14191            .map(|a| a.to_string_lossy().into_owned())
14192            .collect();
14193        assert!(
14194            !none_args.iter().any(|a| a == SUBC_ARG),
14195            "protocol:none argv must not carry --subc; got {none_args:?}"
14196        );
14197        let none_has_nonce = none
14198            .as_std()
14199            .get_envs()
14200            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
14201        assert!(
14202            !none_has_nonce,
14203            "protocol:none spawn must not receive a launch nonce"
14204        );
14205        let none_has_module_id = none
14206            .as_std()
14207            .get_envs()
14208            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
14209        assert!(
14210            none_has_module_id,
14211            "SUBC_MODULE_ID is inert and stays on every path"
14212        );
14213        assert!(
14214            handle.spawn_nonce(&none_spec.module_id).is_none(),
14215            "no nonce record for a process that will never present one"
14216        );
14217
14218        // Control: the subc-wire path is unchanged by the branch above.
14219        let wire_spec = spec(Vec::new());
14220        let mut wire = Command::new("/nonexistent");
14221        let wire_handoff =
14222            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
14223                .expect("subc-wire spawn args apply");
14224        let wire_fd_env = wire
14225            .as_std()
14226            .get_envs()
14227            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
14228            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14229        #[cfg(unix)]
14230        assert_eq!(
14231            wire_fd_env,
14232            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
14233            "a subc-wire spawn names the pipe it will receive at descriptor 3"
14234        );
14235        #[cfg(windows)]
14236        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
14237        #[cfg(not(any(unix, windows)))]
14238        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
14239        let wire_args: Vec<String> = wire
14240            .as_std()
14241            .get_args()
14242            .map(|a| a.to_string_lossy().into_owned())
14243            .collect();
14244        assert_eq!(
14245            wire_args,
14246            vec![
14247                SUBC_ARG.to_string(),
14248                connection_file.to_string_lossy().into_owned()
14249            ],
14250            "a subc-wire spawn still carries --subc <path>"
14251        );
14252        assert_eq!(
14253            wire.as_std()
14254                .get_envs()
14255                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
14256            !cfg!(unix),
14257            "only Windows supplies the environment nonce"
14258        );
14259        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
14260    }
14261
14262    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
14263    /// spec tries to set it; only a swap candidate carries it.
14264    ///
14265    /// "Set it only on candidates" is not enough, because spawn applies the
14266    /// spec's env verbatim and the daemon's own environment is inherited: either
14267    /// could hand a plain restart the swap role, and a module reading it would
14268    /// warm on its long swap budget while callers wait. Asserted as an explicit
14269    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
14270    /// test above gives.
14271    #[test]
14272    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
14273        let role = |command: &Command| {
14274            command
14275                .as_std()
14276                .get_envs()
14277                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
14278                .last()
14279                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
14280        };
14281        let forged = spec(vec![(
14282            SUBC_SPAWN_ROLE_ENV.to_string(),
14283            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
14284        )]);
14285
14286        let mut plain = Command::new("/nonexistent");
14287        apply_child_env(&mut plain, &forged);
14288        apply_spawn_role(&mut plain, SpawnRole::Plain);
14289        assert_eq!(
14290            role(&plain),
14291            Some(None),
14292            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
14293        );
14294
14295        let mut candidate = Command::new("/nonexistent");
14296        apply_child_env(&mut candidate, &spec(Vec::new()));
14297        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
14298        assert_eq!(
14299            role(&candidate),
14300            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
14301        );
14302    }
14303
14304    /// Daemon-private capture retention keys never reach the child.
14305    ///
14306    /// cortexkit-log exposes retention as a Rust struct with no environment
14307    /// names, so these entries are supervisor metadata. Passing them through
14308    /// would invent a public child-process contract by accident.
14309    #[test]
14310    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
14311        let mut command = Command::new("/nonexistent");
14312        apply_child_env(
14313            &mut command,
14314            &spec(vec![
14315                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
14316                ("KEPT".to_string(), "yes".to_string()),
14317            ]),
14318        );
14319        let keys: Vec<String> = command
14320            .as_std()
14321            .get_envs()
14322            .filter(|(_, value)| value.is_some())
14323            .map(|(key, _)| key.to_string_lossy().into_owned())
14324            .collect();
14325        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
14326        assert!(
14327            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
14328            "daemon-private capture key leaked to the child: {keys:?}"
14329        );
14330    }
14331}
14332
14333#[cfg(test)]
14334mod jitter_tests {
14335    use super::jittered_health_delay;
14336    use std::{collections::HashSet, time::Duration};
14337
14338    /// Module ids drawn from a real fleet, so the dispersal claim is about names
14339    /// that actually occur rather than invented ones.
14340    ///
14341    /// This is a SAMPLE, not a registry: the property under test is that distinct
14342    /// ids disperse, which holds for any set of distinct strings. Several entries
14343    /// are already historical (modules get renamed), and that costs nothing here --
14344    /// but it means a reader must not mistake this for the live module set, and a
14345    /// rename sweep will match it without there being anything to change.
14346    const FLEET: [&str; 14] = [
14347        "aft",
14348        "alfonso-core",
14349        "magic-context",
14350        "broca",
14351        "thalamus",
14352        "quota",
14353        "engram",
14354        "plexus",
14355        "cerebellum",
14356        "astrocyte",
14357        "synapse",
14358        "subc-mcp",
14359        "cortexkit-credentials",
14360        "subc-federation",
14361    ];
14362
14363    /// Probes must not converge after a fleet-wide restart.
14364    ///
14365    /// This is the property the jitter exists for: every module reconnects at
14366    /// once, and without dispersal all fourteen would then probe on the same
14367    /// tick forever. Nothing failed visibly when this went untested -- a
14368    /// convergent fleet still probes correctly, just in a burst, so the symptom
14369    /// is a periodic load spike that looks like whatever else is running.
14370    #[test]
14371    fn probe_delays_disperse_across_the_fleet() {
14372        let cadence = Duration::from_secs(30);
14373        let delays: HashSet<Duration> = FLEET
14374            .iter()
14375            .map(|id| jittered_health_delay(id, 0, cadence))
14376            .collect();
14377        assert_eq!(
14378            delays.len(),
14379            FLEET.len(),
14380            "every supervised module must land on its own probe offset"
14381        );
14382    }
14383
14384    /// The offset may only ever DELAY a probe, never bring it forward.
14385    ///
14386    /// A delay below the cadence would probe a module more often than
14387    /// configured, which is the opposite of what an operator asked for and
14388    /// would tighten the failure budget without anyone changing it.
14389    #[test]
14390    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
14391        let cadence = Duration::from_secs(30);
14392        let span = cadence / 10;
14393        for id in FLEET {
14394            for probe_index in 0..8 {
14395                let delay = jittered_health_delay(id, probe_index, cadence);
14396                assert!(
14397                    delay >= cadence,
14398                    "{id}#{probe_index}: jitter must not shorten the cadence"
14399                );
14400                assert!(
14401                    delay < cadence + span,
14402                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
14403                );
14404            }
14405        }
14406    }
14407
14408    /// A module keeps its offset across daemon restarts.
14409    ///
14410    /// The delay is derived rather than randomised precisely so a restart does
14411    /// not re-roll every module into a fresh chance of collision. A random
14412    /// source would satisfy the dispersal test above and quietly lose this.
14413    #[test]
14414    fn a_module_offset_is_stable_across_restarts() {
14415        let cadence = Duration::from_secs(30);
14416        for id in FLEET {
14417            assert_eq!(
14418                jittered_health_delay(id, 0, cadence),
14419                jittered_health_delay(id, 0, cadence),
14420                "{id}: the same module and probe index must produce the same offset"
14421            );
14422        }
14423    }
14424
14425    /// A zero cadence disables probing rather than producing a busy loop.
14426    #[test]
14427    fn zero_cadence_yields_zero_delay() {
14428        assert_eq!(
14429            jittered_health_delay("aft", 0, Duration::ZERO),
14430            Duration::ZERO
14431        );
14432    }
14433}
14434
14435#[cfg(all(test, target_os = "linux"))]
14436mod cgroup_placement_tests {
14437    use super::{
14438        apply_cgroup_placement, remove_module_cgroup, ModuleProtocol, ModuleSpec, SuperviseError,
14439        SupervisedChild,
14440    };
14441    use crate::stderr_tail::{StderrRing, StderrTailConfig};
14442    use cortexkit_test_support::ScratchDir;
14443    use std::{
14444        fs, io,
14445        path::{Path, PathBuf},
14446        sync::{Arc, Mutex},
14447    };
14448    use tokio::process::Command;
14449
14450    #[tokio::test]
14451    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
14452        use super::*;
14453        let dir = ScratchDir::new("unique-spawn-cgroups");
14454        let root = PathBuf::from(format!(
14455            "/sys/fs/cgroup/subc-unique-test-{}-{}",
14456            std::process::id(),
14457            unix_ms_now()
14458        ));
14459        if let Err(error) = fs::create_dir(&root) {
14460            assert!(
14461                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
14462                "required cgroup test cannot execute: {error}"
14463            );
14464            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
14465            return;
14466        }
14467        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
14468        let group_count = || {
14469            fs::read_dir(root.join("subc-modules"))
14470                .unwrap()
14471                .map(|entry| entry.unwrap().file_type().unwrap())
14472                .filter(|kind| kind.is_dir())
14473                .count()
14474        };
14475        let supervisor =
14476            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
14477                .with_cgroup_placement(Some(placement.clone()));
14478        let runtime = supervisor.runtime_config();
14479        let mut spec = ModuleSpec {
14480            module_id: "unique-spawn".into(),
14481            program: PathBuf::from("/bin/sleep"),
14482            args: vec!["60".into()],
14483            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14484                .into_iter()
14485                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
14486                .collect(),
14487            reserved: false,
14488            reserved_prefixes: vec![],
14489            protocol: ModuleProtocol::None,
14490            overlap: Default::default(),
14491        };
14492        let spawn = |spec: &ModuleSpec| {
14493            spawn_child(
14494                spec,
14495                None,
14496                None,
14497                &runtime.stderr_ring,
14498                None,
14499                &runtime.child_roster,
14500                Some(&placement),
14501            )
14502            .unwrap()
14503        };
14504        let mut live = spawn(&spec);
14505        for _ in 0..3 {
14506            // A new process can enter the old slot while retirement is pending.
14507            let next = spawn(&spec);
14508            assert_ne!(live.module_id, next.module_id);
14509            live.start_kill().unwrap();
14510            live.wait().await.unwrap();
14511            live = next;
14512            assert!(
14513                live.child.try_wait().unwrap().is_none(),
14514                "retiring the old slot must not kill the replacement"
14515            );
14516            assert_eq!(
14517                group_count(),
14518                1,
14519                "only the live spawn's cgroup should remain"
14520            );
14521        }
14522        supervisor.begin_daemon_shutdown();
14523        let reap = tokio::spawn(async move {
14524            live.wait().await.unwrap();
14525        });
14526        supervisor
14527            .end_children_for_daemon_shutdown(false, std::future::pending())
14528            .await;
14529        reap.await.unwrap();
14530        assert_eq!(group_count(), 0);
14531        // A normal exit uses the same tree-cleanup path as a killed spawn.
14532        spec.program = PathBuf::from("/bin/true");
14533        spec.args.clear();
14534        let fresh_roster = ChildRoster::default();
14535        let mut short = spawn_child(
14536            &spec,
14537            None,
14538            None,
14539            &runtime.stderr_ring,
14540            None,
14541            &fresh_roster,
14542            Some(&placement),
14543        )
14544        .unwrap();
14545        short.wait().await.unwrap();
14546        assert_eq!(group_count(), 0);
14547        spec.module_id = "_".repeat(255);
14548        let mut long_id = spawn_child(
14549            &spec,
14550            None,
14551            None,
14552            &runtime.stderr_ring,
14553            None,
14554            &fresh_roster,
14555            Some(&placement),
14556        )
14557        .unwrap();
14558        long_id.wait().await.unwrap();
14559        assert_eq!(
14560            group_count(),
14561            0,
14562            "valid long module IDs must not exceed cgroup NAME_MAX"
14563        );
14564        fs::remove_dir(root.join("subc-modules")).unwrap();
14565        fs::remove_dir(root).unwrap();
14566        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
14567    }
14568
14569    #[test]
14570    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
14571        let path = Path::new("/definitely-missing-subc-cgroup");
14572        let mut command = Command::new("true");
14573        let error = apply_cgroup_placement(
14574            &mut command,
14575            &ModuleSpec {
14576                module_id: "broken-cgroup".to_string(),
14577                program: PathBuf::from("true"),
14578                args: Vec::new(),
14579                env: Vec::new(),
14580                reserved: false,
14581                reserved_prefixes: Vec::new(),
14582                protocol: ModuleProtocol::Subc,
14583                overlap: Default::default(),
14584            },
14585            path,
14586        )
14587        .expect_err("a parent cgroup open failure must reject the supervised spawn");
14588        let reason = error.to_string();
14589
14590        assert!(
14591            matches!(error, SuperviseError::Cgroup { .. }),
14592            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
14593        );
14594        assert!(
14595            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
14596            "parent cgroup open failure must name cgroup.procs: {reason}"
14597        );
14598    }
14599
14600    #[tokio::test]
14601    async fn reaping_a_child_removes_its_empty_module_cgroup() {
14602        let root = ScratchDir::new("supervisor-reap-cgroup");
14603        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
14604        let placement = subc_cgroup::prepare_at(&root)
14605            .expect("prepare scratch cgroup root")
14606            .expect("scratch root has a cgroup.procs marker");
14607        let module_id = "reaped-module";
14608        let module = placement
14609            .module_path(module_id)
14610            .expect("create scratch module cgroup");
14611        let child = Command::new("true")
14612            .env("XDG_DATA_HOME", root.path())
14613            .env("XDG_RUNTIME_DIR", root.path())
14614            .env("XDG_CONFIG_HOME", root.path())
14615            .spawn()
14616            .expect("spawn short-lived child");
14617        let pid = child.id().expect("spawned child has pid");
14618        let mut child = SupervisedChild {
14619            child,
14620            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
14621            protocol: ModuleProtocol::Subc,
14622            module_id: module_id.to_string(),
14623            cgroup_placement: Some(placement),
14624            stdout_pump: None,
14625            stderr_pump: None,
14626            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
14627            spawned_at_ms: 0,
14628            spawned_from: PathBuf::from("true"),
14629            spawned_file_identity: None,
14630            process_start_time: None,
14631            process_identity: None,
14632            pid,
14633            roster_guard: None,
14634            #[cfg(target_os = "macos")]
14635            privacy_exec: None,
14636            spawn_failure: None,
14637        };
14638
14639        child.wait().await.expect("reap short-lived child");
14640
14641        assert!(
14642            !module.exists(),
14643            "reaping the supervised child must remove its empty cgroup"
14644        );
14645    }
14646
14647    #[test]
14648    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
14649        let root = ScratchDir::new("supervisor-non-empty-cgroup");
14650        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
14651        let placement = subc_cgroup::prepare_at(&root)
14652            .expect("prepare scratch cgroup root")
14653            .expect("scratch root has a cgroup.procs marker");
14654        let module = placement
14655            .module_path("surviving-module")
14656            .expect("create scratch module cgroup");
14657        fs::write(module.join("surviving-process"), b"still present")
14658            .expect("make scratch cgroup non-empty");
14659        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
14660
14661        remove_module_cgroup(&placement, "surviving-module");
14662
14663        let logs = crate::router::test_log::captured_logs(&logs);
14664        assert!(
14665            module.exists(),
14666            "failed removal must leave the cgroup intact"
14667        );
14668        assert!(
14669            logs.contains("could not remove module cgroup after process exit; continuing teardown")
14670                && logs.contains("surviving-module"),
14671            "best-effort removal must report the failure without returning it: {logs}"
14672        );
14673    }
14674
14675    #[test]
14676    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
14677        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
14678        let reason = SuperviseError::Spawn {
14679            program: PathBuf::from("/bin/true"),
14680            source: io::Error::from_raw_os_error(13),
14681            cgroup_path: Some(cgroup_path.clone()),
14682        }
14683        .to_string();
14684
14685        assert!(
14686            reason.contains(&cgroup_path.display().to_string()),
14687            "a pre_exec spawn failure must name the cgroup path: {reason}"
14688        );
14689    }
14690}
14691
14692#[cfg(test)]
14693mod spawn_subscriber_lag_tests {
14694    use super::*;
14695
14696    /// A subscriber whose connection stops draining is dropped once its frame
14697    /// channel fills. The client must learn that from a terminal Error frame
14698    /// after the frames already queued for it, not from a stream that simply
14699    /// goes quiet.
14700    #[tokio::test]
14701    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
14702        let feed = SpawnEventFeed::default();
14703        feed.configure_incarnation("lag-incarnation".to_string());
14704        // A one-slot connection queue that nobody reads until the emits are
14705        // done: the forwarder parks on it and the subscriber channel fills.
14706        let (tx, mut rx) = mpsc::channel(1);
14707        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
14708            .expect("subscribe");
14709        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
14710        for index in 0..emitted {
14711            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
14712            // Let the forwarder take what it can so the fill point is the
14713            // subscriber channel, not a scheduling accident.
14714            tokio::task::yield_now().await;
14715        }
14716        assert_eq!(
14717            feed.subscriber_count(),
14718            0,
14719            "the lagged subscriber must be removed"
14720        );
14721
14722        let mut data = Vec::new();
14723        let mut last = None;
14724        loop {
14725            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14726                .await
14727                .expect("the forwarder must finish once the subscriber is dropped");
14728            let Some(outbound) = next else { break };
14729            let frame = outbound.frame;
14730            if frame.header.ty == FrameType::StreamData {
14731                assert!(last.is_none(), "no data may follow the terminal frame");
14732                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
14733                data.push(event.cursor.seq);
14734            } else {
14735                assert!(last.is_none(), "exactly one terminal frame");
14736                last = Some(frame);
14737            }
14738        }
14739        assert!(!data.is_empty(), "queued frames drain before the terminal");
14740        for pair in data.windows(2) {
14741            assert_eq!(
14742                pair[1],
14743                pair[0] + 1,
14744                "queued frames arrive dense and in order"
14745            );
14746        }
14747        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
14748        assert_eq!(terminal.header.ty, FrameType::Error);
14749        assert_eq!(terminal.header.corr, 7);
14750        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
14751        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
14752        let detail = body.detail.expect("lagged error carries detail");
14753        assert_eq!(
14754            detail["first_undelivered_cursor"]["seq"],
14755            data.last().unwrap() + 1,
14756            "the named cursor is the first event the subscriber did not receive"
14757        );
14758        assert_eq!(
14759            detail["first_undelivered_cursor"]["daemon_incarnation"],
14760            "lag-incarnation"
14761        );
14762    }
14763}
14764
14765#[cfg(test)]
14766mod terminal_history_read_concurrency_tests {
14767    use super::*;
14768    use crate::terminal_journal::read_pause;
14769    use cortexkit_test_support::ScratchDir;
14770    use std::sync::mpsc as std_mpsc;
14771
14772    fn journaled_ring(
14773        journal: &Arc<crate::terminal_journal::TerminalJournal>,
14774    ) -> Arc<Mutex<TerminalRing>> {
14775        Arc::new(Mutex::new(
14776            TerminalRing::new(TerminalRingConfig::default(), 1)
14777                .with_journal(Some(Arc::clone(journal))),
14778        ))
14779    }
14780
14781    fn crash(at_ms: u64) -> ExitReport {
14782        ExitReport {
14783            kind: ExitKind::Crash,
14784            code: Some(1),
14785            signal: None,
14786            at_ms,
14787        }
14788    }
14789
14790    /// Record an exit on another thread and report whether it finished within
14791    /// `bound`. The recorder thread is left running if it did not.
14792    fn record_within(
14793        module_id: &'static str,
14794        ring: &Arc<Mutex<TerminalRing>>,
14795        at_ms: u64,
14796        bound: Duration,
14797    ) -> bool {
14798        let ring = Arc::clone(ring);
14799        let (done, done_rx) = std_mpsc::channel();
14800        std::thread::spawn(move || {
14801            record_terminal(
14802                module_id,
14803                &ring,
14804                &SpawnEventFeed::default(),
14805                &crash(at_ms),
14806                TerminalDisposition::Restarting,
14807            );
14808            let _ = done.send(());
14809        });
14810        done_rx.recv_timeout(bound).is_ok()
14811    }
14812
14813    /// A history read in progress must not hold the journal writer (which every
14814    /// module's exit recording needs) or the module's own ring. Exits recorded
14815    /// while the read is paused complete promptly; the paused read answers as of
14816    /// the moment it started, and the next read has each exit exactly once.
14817    #[test]
14818    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
14819        let dir = ScratchDir::new("terminal-history-concurrent-read");
14820        let path = dir.join("terminals.jsonl");
14821        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
14822            path.clone(),
14823            "daemon".into(),
14824        ));
14825        let reader_ring = journaled_ring(&journal);
14826        let other_ring = journaled_ring(&journal);
14827        assert!(record_within(
14828            "reader-module",
14829            &reader_ring,
14830            10,
14831            Duration::from_secs(5)
14832        ));
14833
14834        let (started, release) = read_pause::install(&path);
14835        let reading = {
14836            let ring = Arc::clone(&reader_ring);
14837            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
14838        };
14839        started
14840            .recv_timeout(Duration::from_secs(5))
14841            .expect("the history read reached its pause");
14842
14843        let bound = Duration::from_secs(1);
14844        assert!(
14845            record_within("other-module", &other_ring, 20, bound),
14846            "another module's exit waited on a history read (journal writer held)"
14847        );
14848        assert!(
14849            record_within("reader-module", &reader_ring, 30, bound),
14850            "the read module's own exit waited on its history read (ring held)"
14851        );
14852
14853        drop(release);
14854        let paused = reading.join().unwrap();
14855        assert_eq!(
14856            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
14857            vec![10],
14858            "an exit recorded after the read began lands in neither half of it"
14859        );
14860        assert_eq!(paused.journal_skipped_lines, 0);
14861        assert_eq!(paused.journal_read_errors, 0);
14862
14863        let after = durable_terminal_history_of(&reader_ring, "reader-module");
14864        assert_eq!(
14865            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
14866            vec![10, 30],
14867            "the next read merges ring and journal with no duplicate"
14868        );
14869        assert_eq!(after.journal_skipped_lines, 0);
14870    }
14871}
14872
14873/// What a restart does with the exited process's output readers. These drive
14874/// the same settle functions the supervisor calls, with a reader the test
14875/// holds, so a reader that has not been scheduled by the bound is a controlled
14876/// input rather than something only a loaded machine produces.
14877#[cfg(test)]
14878mod stderr_settle_tests {
14879    use std::{
14880        future::Future,
14881        io,
14882        pin::Pin,
14883        sync::{Arc, Mutex},
14884        task::{Context, Poll},
14885        time::Duration,
14886    };
14887
14888    use tokio::{
14889        io::{AsyncRead, ReadBuf},
14890        sync::oneshot,
14891        time::Instant,
14892    };
14893
14894    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
14895    use crate::stderr_tail::{
14896        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
14897        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
14898    };
14899
14900    const BOUND: Duration = Duration::from_millis(250);
14901
14902    /// Yields `before`, then stays pending until the gate is released, then
14903    /// yields `after` and reaches EOF. The bytes after the gate were written
14904    /// by a process that has already exited; only the reader is behind.
14905    struct HeldReader {
14906        before: Option<Vec<u8>>,
14907        gate: Option<oneshot::Receiver<()>>,
14908        after: io::Cursor<Vec<u8>>,
14909    }
14910
14911    impl AsyncRead for HeldReader {
14912        fn poll_read(
14913            mut self: Pin<&mut Self>,
14914            cx: &mut Context<'_>,
14915            buf: &mut ReadBuf<'_>,
14916        ) -> Poll<io::Result<()>> {
14917            if let Some(bytes) = self.before.take() {
14918                buf.put_slice(&bytes);
14919                return Poll::Ready(Ok(()));
14920            }
14921            if let Some(gate) = self.gate.as_mut() {
14922                match Pin::new(gate).poll(cx) {
14923                    Poll::Pending => return Poll::Pending,
14924                    Poll::Ready(_) => self.gate = None,
14925                }
14926            }
14927            Pin::new(&mut self.after).poll_read(cx, buf)
14928        }
14929    }
14930
14931    struct DiscardSink;
14932
14933    impl OutputSink for DiscardSink {
14934        fn write_line(&mut self, _line: &[u8]) {}
14935    }
14936
14937    fn line(text: &str) -> TailEntry {
14938        TailEntry::Line {
14939            text: text.to_string(),
14940            truncated: false,
14941            at_ms: None,
14942        }
14943    }
14944
14945    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
14946        ring.lock().unwrap()
14947    }
14948
14949    /// Start a reader for a new process generation that delivers `before`
14950    /// immediately and `after` only once the returned sender fires (or is
14951    /// dropped).
14952    fn held_pump(
14953        ring: &Arc<Mutex<StderrRing>>,
14954        before: &str,
14955        after: &str,
14956    ) -> (StderrPump, oneshot::Sender<()>) {
14957        let generation = lock(ring).begin_process();
14958        let (release, gate) = oneshot::channel();
14959        let reader = HeldReader {
14960            before: Some(before.as_bytes().to_vec()),
14961            gate: Some(gate),
14962            after: io::Cursor::new(after.as_bytes().to_vec()),
14963        };
14964        let task = tokio::spawn(pump_stderr_to(
14965            reader,
14966            Arc::clone(ring),
14967            generation,
14968            DiscardSink,
14969        ));
14970        (StderrPump { task, generation }, release)
14971    }
14972
14973    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
14974        for _ in 0..1000 {
14975            if done(&lock(ring)) {
14976                return;
14977            }
14978            tokio::time::sleep(Duration::from_millis(1)).await;
14979        }
14980        panic!(
14981            "ring never reached the expected state: {:?}",
14982            lock(ring).snapshot(None, None)
14983        );
14984    }
14985
14986    #[tokio::test(start_paused = true)]
14987    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
14988        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
14989        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
14990
14991        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
14992        let before_release = lock(&ring).snapshot(None, None);
14993        assert!(
14994            matches!(before_release.capture, CaptureState::Incomplete { .. }),
14995            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
14996        );
14997
14998        // The restart: the next process starts and writes before the old
14999        // reader catches up.
15000        let next = lock(&ring).begin_process();
15001        lock(&ring).push_line_from(next, "next process booting");
15002        release.send(()).unwrap();
15003        wait_until(&ring, |ring| {
15004            ring.snapshot(None, None).capture == CaptureState::Captured
15005        })
15006        .await;
15007
15008        assert_eq!(
15009            untimed(lock(&ring).snapshot(None, None).entries),
15010            vec![
15011                line("booting"),
15012                line("config error: missing storage"),
15013                TailEntry::ProcessStart,
15014                line("next process booting"),
15015            ],
15016            "the crash's last line must survive a slow reader and stay in the crashed process's section"
15017        );
15018    }
15019
15020    #[tokio::test(start_paused = true)]
15021    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
15022    ) {
15023        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15024        // `_held` is never fired: a descendant keeps the pipe open for the
15025        // whole test.
15026        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
15027
15028        let started = Instant::now();
15029        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
15030        assert_eq!(
15031            started.elapsed(),
15032            BOUND,
15033            "the restart must wait exactly the bound for a pipe that stays open, no longer"
15034        );
15035
15036        let next = lock(&ring).begin_process();
15037        lock(&ring).push_line_from(next, "next process booting");
15038        tokio::time::sleep(Duration::from_secs(60)).await;
15039
15040        let snapshot = lock(&ring).snapshot(None, None);
15041        match &snapshot.capture {
15042            CaptureState::Incomplete { reason } => assert!(
15043                reason.contains("had not reached EOF") && reason.contains("250ms"),
15044                "the reason must say what is missing and after how long: {reason}"
15045            ),
15046            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
15047        }
15048        assert_eq!(
15049            untimed(snapshot.entries),
15050            vec![
15051                line("parent exiting"),
15052                TailEntry::ProcessStart,
15053                line("next process booting"),
15054            ]
15055        );
15056    }
15057
15058    #[tokio::test(start_paused = true)]
15059    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
15060        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15061        let (pump, release) = held_pump(&ring, "one\n", "two\n");
15062        release.send(()).unwrap();
15063
15064        settle_stderr_pump("clean", &ring, pump, BOUND).await;
15065
15066        let snapshot = lock(&ring).snapshot(None, None);
15067        assert_eq!(snapshot.capture, CaptureState::Captured);
15068        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
15069    }
15070
15071    #[tokio::test(start_paused = true)]
15072    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
15073        const BURST: usize = 150;
15074        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
15075        let path = capture.join("two-pipe.stderr.log");
15076        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
15077        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15078        let generation = lock(&ring).begin_process();
15079        let stdout = (0..BURST)
15080            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
15081            .collect::<String>();
15082        let stderr = (0..BURST)
15083            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
15084            .collect::<String>();
15085        // Model a child that has written both bursts and exited. Hold the stdout
15086        // reader after one line and a partial prefix, as if pipe readiness or
15087        // capture-file writes had delayed it. The final line also needs EOF to
15088        // flush: neither complete nor partial buffered output may be discarded.
15089        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
15090        let (release, gate) = oneshot::channel();
15091        let reader = HeldReader {
15092            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
15093            gate: Some(gate),
15094            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
15095        };
15096        let (finished, completion) = oneshot::channel();
15097        let stdout_sink = sink.clone();
15098        let pump = tokio::spawn(async move {
15099            pump_stdout_to(reader, stdout_sink).await;
15100            finished.send(()).unwrap();
15101        });
15102        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
15103
15104        let started = Instant::now();
15105        settle_stdout_pump("two-pipe", pump, BOUND).await;
15106        assert_eq!(
15107            started.elapsed(),
15108            BOUND,
15109            "a late reader must not hold up restart"
15110        );
15111        assert_eq!(
15112            std::fs::read_to_string(&path).unwrap().lines().count(),
15113            BURST + 1
15114        );
15115
15116        release
15117            .send(())
15118            .expect("stdout reader must survive the exit drain bound");
15119        tokio::time::timeout(Duration::from_secs(1), completion)
15120            .await
15121            .expect("released stdout reader must reach EOF")
15122            .expect("stdout capture must finish at EOF, not be cancelled");
15123        let contents = std::fs::read_to_string(&path).unwrap();
15124        let mut lines = contents
15125            .lines()
15126            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
15127            .collect::<Vec<_>>();
15128        let mut expected = (0..BURST)
15129            .flat_map(|i| {
15130                [
15131                    format!("out-{i:04}-{}", "o".repeat(64)),
15132                    format!("err-{i:04}-{}", "e".repeat(64)),
15133                ]
15134            })
15135            .collect::<Vec<_>>();
15136        lines.sort_unstable();
15137        expected.sort_unstable();
15138        assert_eq!(
15139            lines, expected,
15140            "both pipes must retain every complete line exactly once"
15141        );
15142    }
15143}
15144
15145/// Containment of a module's process tree (issue #109).
15146///
15147/// The behaviour these defend against is a module helper surviving its module:
15148/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
15149/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
15150/// compounds it.
15151///
15152/// They run against the SUPERVISOR rather than the job-object crate because the
15153/// claim is about teardown: a crate-level test proves a job can reap a tree, not
15154/// that the daemon's drain path reaches it.
15155///
15156/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
15157/// lane there is a separate containment path with its own tests.
15158#[cfg(all(test, windows))]
15159mod job_containment_tests {
15160    use super::*;
15161    use cortexkit_test_support::ScratchDir;
15162    use std::{
15163        path::{Path, PathBuf},
15164        sync::{Arc, Mutex},
15165        time::{Duration, Instant},
15166    };
15167
15168    /// The stub, expected beside this test executable.
15169    ///
15170    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
15171    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
15172    /// failure then reads as a broken test rather than an unbuilt dependency.
15173    fn stub_path() -> PathBuf {
15174        let mut path = std::env::current_exe().expect("current_exe available in tests");
15175        path.pop();
15176        path.pop();
15177        path.push("fake-aft-stub.exe");
15178        assert!(
15179            path.exists(),
15180            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
15181             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
15182            path.display()
15183        );
15184        path
15185    }
15186
15187    /// Poll for the grandchild pid the stub records, and parse it.
15188    fn read_grandchild_pid(path: &Path) -> u32 {
15189        let deadline = Instant::now() + Duration::from_secs(10);
15190        loop {
15191            if let Ok(contents) = std::fs::read_to_string(path) {
15192                if let Ok(pid) = contents.trim().parse() {
15193                    return pid;
15194                }
15195            }
15196            assert!(
15197                Instant::now() < deadline,
15198                "the stub never recorded a grandchild pid at {}",
15199                path.display()
15200            );
15201            std::thread::sleep(Duration::from_millis(10));
15202        }
15203    }
15204
15205    /// Everything one fixture run needs, so the two tests below differ in exactly
15206    /// one place: whether the child is contained.
15207    struct Fixture {
15208        _dir: ScratchDir,
15209        module_id: String,
15210        grandchild: u32,
15211        child: Option<SupervisedChild>,
15212        registry: Arc<Registry>,
15213        snapshot: Arc<Mutex<SupervisorSnapshot>>,
15214        terminal_ring: Arc<Mutex<TerminalRing>>,
15215        spawn_events: SpawnEventFeed,
15216    }
15217
15218    fn fixture(label: &str, module_id: &str) -> Fixture {
15219        let dir = ScratchDir::new(label);
15220        let pid_file = dir.join("grandchild.pid");
15221        let supervisor = Supervisor::new_for_test(
15222            Arc::new(Registry::default()),
15223            RestartPolicy::new(3, Duration::ZERO),
15224        );
15225        let runtime = supervisor.runtime_config();
15226        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15227        let spec = ModuleSpec {
15228            module_id: module_id.to_string(),
15229            program: stub_path(),
15230            // Zero args deliberately: a `--subc` argument would make the stub dial
15231            // a daemon that is not there, and the failure would land in the same
15232            // stderr ring this fixture exists to keep quiet.
15233            args: Vec::new(),
15234            env: vec![
15235                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
15236                (
15237                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
15238                    pid_file.display().to_string(),
15239                ),
15240            ],
15241            reserved: false,
15242            reserved_prefixes: Vec::new(),
15243            protocol: ModuleProtocol::Subc,
15244            overlap: Default::default(),
15245        };
15246        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
15247            .expect("spawn the supervised fixture");
15248        let grandchild = read_grandchild_pid(&pid_file);
15249        Fixture {
15250            _dir: dir,
15251            module_id: module_id.to_string(),
15252            grandchild,
15253            child: Some(child),
15254            registry: Arc::new(Registry::default()),
15255            snapshot,
15256            terminal_ring: Arc::clone(&runtime.terminal_ring),
15257            spawn_events: SpawnEventFeed::default(),
15258        }
15259    }
15260
15261    impl Fixture {
15262        /// Drain through the supervisor's own teardown path.
15263        async fn drain(&mut self) {
15264            let child = self
15265                .child
15266                .take()
15267                .expect("the fixture child is still present");
15268            drain_child_to_state(
15269                &self.module_id,
15270                ModuleProtocol::Subc,
15271                // No forwarding table in this fixture, so nothing reaches the
15272                // child over a connection.
15273                StopNotice::NotSent,
15274                &self.registry,
15275                None,
15276                &self.snapshot,
15277                &self.terminal_ring,
15278                &self.spawn_events,
15279                child,
15280                Duration::from_millis(500),
15281                ModuleState::Stopped,
15282                Some(false),
15283            )
15284            .await
15285            .expect("drain the supervised fixture");
15286        }
15287    }
15288
15289    /// Teardown reaps the grandchild, not merely the direct child.
15290    ///
15291    /// This is the assertion the change exists for. Before containment the
15292    /// grandchild survived: it is a separate process, and `start_kill` is
15293    /// `TerminateProcess` scoped to one pid.
15294    #[tokio::test]
15295    async fn teardown_reaps_the_grandchild() {
15296        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
15297        let grandchild = fixture.grandchild;
15298
15299        assert!(
15300            subc_jobobject::process_exists(grandchild),
15301            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
15302        );
15303
15304        fixture.drain().await;
15305
15306        assert!(
15307            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
15308            "grandchild {grandchild} outlived module teardown: the tree was not contained"
15309        );
15310    }
15311
15312    /// The mutation control: with containment withheld, the grandchild survives
15313    /// the same kill.
15314    ///
15315    /// This is the defect reproduction from #109 — a direct-child kill reaches
15316    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
15317    /// supervisor because `spawn_and_mark_running` now always contains on
15318    /// Windows, which is the point: there is no longer a path that spawns
15319    /// uncontained, so the control has to construct one.
15320    ///
15321    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
15322    /// grandchild ever dies here, that test is passing for a reason unrelated to
15323    /// the job object and the containment claim is unproven.
15324    #[test]
15325    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
15326        let dir = ScratchDir::new("teardown-uncontained");
15327        let pid_file = dir.join("grandchild.pid");
15328        let mut child = std::process::Command::new(stub_path())
15329            .env("FAKE_AFT_NEVER_CONNECT", "1")
15330            .env(
15331                "FAKE_AFT_GRANDCHILD_PID_FILE",
15332                pid_file.display().to_string(),
15333            )
15334            .stdin(std::process::Stdio::null())
15335            .stdout(std::process::Stdio::null())
15336            .stderr(std::process::Stdio::null())
15337            .spawn()
15338            .expect("spawn the uncontained fixture");
15339        let grandchild = read_grandchild_pid(&pid_file);
15340
15341        // Exactly what the pre-fix teardown did: kill the direct child.
15342        child.kill().expect("kill the direct child");
15343        let _ = child.wait();
15344
15345        assert!(
15346            subc_jobobject::process_exists(grandchild),
15347            "grandchild {grandchild} died with the direct child, so this control no longer \
15348             distinguishes contained from uncontained teardown and the regression test is \
15349             passing vacuously"
15350        );
15351
15352        // The orphan this control demonstrates is the leak the fix prevents, so
15353        // the control must not leave one behind.
15354        kill_tree(grandchild);
15355    }
15356
15357    /// Crash durability: closing the containment handle reaps the tree with no
15358    /// teardown code running at all.
15359    ///
15360    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
15361    /// call anything — and it is why containment is a kernel property of the
15362    /// handle rather than a step in the drain. Discovered by getting the
15363    /// mutation control wrong: clearing `job` to "disable" containment instead
15364    /// killed the tree, which is the guarantee, not a mistake.
15365    #[tokio::test]
15366    async fn dropping_containment_reaps_the_grandchild() {
15367        let mut fixture = fixture("drop-containment", "tree-drop");
15368        let grandchild = fixture.grandchild;
15369
15370        assert!(subc_jobobject::process_exists(grandchild));
15371
15372        // No `drain` call, no kill: dropping the handle is the entire mechanism.
15373        fixture.child.as_mut().expect("child present").job = None;
15374
15375        assert!(
15376            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
15377            "grandchild {grandchild} survived the containment handle closing, so a daemon \
15378             crash would leave the tree behind"
15379        );
15380    }
15381
15382    /// Kill a pid and its tree, then confirm it is gone.
15383    fn kill_tree(pid: u32) {
15384        let _ = std::process::Command::new("taskkill.exe")
15385            .args(["/PID", &pid.to_string(), "/T", "/F"])
15386            .stdin(std::process::Stdio::null())
15387            .stdout(std::process::Stdio::null())
15388            .stderr(std::process::Stdio::null())
15389            .status();
15390        assert!(
15391            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
15392            "could not clean up grandchild {pid}"
15393        );
15394    }
15395}
15396
15397#[cfg(test)]
15398mod privacy_trampoline_configuration_tests {
15399    #[tokio::test]
15400    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15401    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
15402        #[cfg(target_os = "macos")]
15403        {
15404            let supervisor = super::Supervisor::new(
15405                std::sync::Arc::new(crate::Registry::default()),
15406                super::RestartPolicy::default(),
15407            );
15408            let error = supervisor.spawn(spec()).unwrap_err();
15409            assert!(
15410                error
15411                    .to_string()
15412                    .contains("no privacy trampoline configured"),
15413                "{error}"
15414            );
15415        }
15416    }
15417
15418    #[tokio::test]
15419    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15420    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
15421        #[cfg(target_os = "macos")]
15422        {
15423            let supervisor = super::Supervisor::new(
15424                std::sync::Arc::new(crate::Registry::default()),
15425                super::RestartPolicy::default(),
15426            )
15427            .with_privacy_trampoline(std::env::current_exe().unwrap());
15428            let error = supervisor.spawn(spec()).unwrap_err();
15429            assert!(
15430                error
15431                    .to_string()
15432                    .contains("binary does not implement the privacy trampoline protocol"),
15433                "{error}"
15434            );
15435        }
15436    }
15437
15438    #[cfg(target_os = "macos")]
15439    fn spec() -> super::ModuleSpec {
15440        super::ModuleSpec {
15441            module_id: "privacy-configuration".into(),
15442            program: "/bin/sleep".into(),
15443            args: vec!["30".into()],
15444            env: vec![],
15445            reserved: false,
15446            reserved_prefixes: vec![],
15447            protocol: subc_control::ModuleProtocol::None,
15448            overlap: super::ModuleOverlap::Exclusive,
15449        }
15450    }
15451}
15452
15453#[cfg(test)]
15454mod privacy_exec_boundary_tests {
15455    #[cfg(target_os = "macos")]
15456    use super::*;
15457    #[cfg(target_os = "macos")]
15458    use std::{
15459        io::{Read, Write},
15460        net::{TcpListener, TcpStream},
15461    };
15462
15463    /// Unit-test-only pause at the actual early image read, not at a later
15464    /// status read. Production supervisors never inspect this environment key.
15465    #[cfg(target_os = "macos")]
15466    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
15467        if let Some((_, path)) = spec
15468            .env
15469            .iter()
15470            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
15471        {
15472            let mut barrier = TcpStream::connect(path).unwrap();
15473            barrier
15474                .set_read_timeout(Some(Duration::from_secs(30)))
15475                .unwrap();
15476            barrier.write_all(&pid.to_ne_bytes()).unwrap();
15477            let mut release = [0];
15478            barrier.read_exact(&mut release).unwrap();
15479            assert_eq!(&release, b"X");
15480        }
15481    }
15482
15483    #[cfg(target_os = "macos")]
15484    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
15485        ModuleSpec {
15486            module_id: "privacy-boundary".into(),
15487            program: program.into(),
15488            args: args.iter().map(|arg| (*arg).into()).collect(),
15489            env: vec![],
15490            reserved: false,
15491            reserved_prefixes: vec![],
15492            protocol: ModuleProtocol::None,
15493            overlap: ModuleOverlap::Exclusive,
15494        }
15495    }
15496
15497    #[cfg(target_os = "macos")]
15498    async fn accept(listener: TcpListener) -> TcpStream {
15499        // Socket readiness, not elapsed time, establishes both pause points.
15500        let listener = tokio::net::TcpListener::from_std({
15501            listener.set_nonblocking(true).unwrap();
15502            listener
15503        })
15504        .unwrap();
15505        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
15506            .await
15507            .unwrap()
15508            .unwrap();
15509        let stream = stream.into_std().unwrap();
15510        stream.set_nonblocking(false).unwrap();
15511        stream
15512            .set_read_timeout(Some(Duration::from_secs(30)))
15513            .unwrap();
15514        stream
15515    }
15516
15517    #[tokio::test]
15518    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15519    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
15520        #[cfg(target_os = "macos")]
15521        {
15522            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
15523            // Loopback sockets also work when the replay adapter's TMPDIR is
15524            // longer than Darwin's Unix-domain socket path limit.
15525            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
15526            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
15527            let record = root.join("live-children.json");
15528            let supervisor =
15529                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15530                    .with_live_children_record(&record);
15531            let runtime = supervisor.runtime_config();
15532            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15533            let mut spec = spec("/bin/sleep", &["30"]);
15534            spec.env = vec![
15535                (
15536                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
15537                    exec_listener.local_addr().unwrap().to_string(),
15538                ),
15539                (
15540                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
15541                    sample_listener.local_addr().unwrap().to_string(),
15542                ),
15543            ];
15544            let spawn = tokio::task::spawn_blocking(move || {
15545                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
15546            });
15547            let mut sample = accept(sample_listener).await;
15548            let mut pid = [0; 4];
15549            sample.read_exact(&mut pid).unwrap();
15550            let pid = u32::from_ne_bytes(pid);
15551            let mut exec = accept(exec_listener).await;
15552            let mut ready = [0];
15553            exec.read_exact(&mut ready).unwrap();
15554            assert_eq!(&ready, b"R");
15555            // The early read is guaranteed to see a real, non-null trampoline
15556            // image: the fixture has reached its barrier and cannot exec yet.
15557            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
15558            assert_eq!(
15559                observe_spawned_image(pid).unwrap().executable,
15560                Some(trampoline)
15561            );
15562            sample.write_all(b"X").unwrap();
15563            let mut child = spawn.await.unwrap();
15564            let early = crate::live_children::read_record(&record).unwrap();
15565            assert_eq!(early.len(), 1);
15566            assert_eq!(early[0].pid, pid);
15567            assert_eq!(
15568                early[0].executable, None,
15569                "unconfirmed trampoline image entered the roster"
15570            );
15571            assert!(child.report_ready.get().is_none());
15572            // The barrier's duration is unrelated to the production five-second
15573            // exec budget. Start the test's confirmation budget upon release.
15574            child.privacy_exec.as_mut().unwrap().deadline =
15575                tokio::time::Instant::now() + Duration::from_secs(30);
15576            exec.write_all(b"X").unwrap();
15577            child.confirm_privacy_exec().await;
15578            assert_eq!(child.spawn_failure, None);
15579            assert!(child.report_ready.get().is_some());
15580            let confirmed = crate::live_children::read_record(&record).unwrap();
15581            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
15582            assert_ne!(module, trampoline);
15583            assert_eq!(confirmed[0].executable, Some(module.into()));
15584            child.start_kill().unwrap();
15585            child.wait().await.unwrap();
15586            child.release_roster();
15587        }
15588    }
15589
15590    #[tokio::test]
15591    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15592    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
15593        #[cfg(target_os = "macos")]
15594        {
15595            let registry = Arc::new(Registry::default());
15596            let policy = RestartPolicy::new(0, Duration::ZERO);
15597            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
15598            let runtime = supervisor.runtime_config();
15599            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15600            let spec = spec("/bin/sh", &["-c", "exit 121"]);
15601            // Drive spawn and confirmation separately instead of starting the
15602            // monitor. WNOWAIT observes a real exit without consuming its status,
15603            // so confirmation's first try_wait must take the already-exited arm.
15604            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
15605            let pid = child.pid;
15606            tokio::task::spawn_blocking(move || {
15607                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
15608            })
15609            .await
15610            .unwrap()
15611            .unwrap();
15612            child.privacy_exec.as_mut().unwrap().deadline =
15613                tokio::time::Instant::now() + Duration::from_secs(30);
15614            let status = child.wait().await.unwrap();
15615            assert_eq!(status.code(), Some(121));
15616            assert!(child.privacy_exec.is_none());
15617            assert!(
15618                child.report_ready.get().is_none(),
15619                "an exited module must not publish a live pid"
15620            );
15621            let report = classify_reaped_child_exit(&snapshot, &child, &status);
15622            on_child_exit(
15623                &spec,
15624                policy,
15625                &registry,
15626                &snapshot,
15627                &runtime.terminal_ring,
15628                &runtime.spawn_events,
15629                &runtime.child_roster,
15630                report,
15631            )
15632            .await;
15633            let state = lock_snapshot(&snapshot).unwrap();
15634            assert_eq!(state.state, ModuleState::Failed);
15635            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
15636            assert_eq!(state.reported_pid(), None);
15637            drop(state);
15638            let history = runtime.terminal_ring.lock().unwrap().snapshot();
15639            assert_eq!(history.entries.len(), 1);
15640            let terminal = &history.entries[0];
15641            assert_eq!(terminal.exit_code, Some(121));
15642            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
15643            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
15644            assert_eq!(
15645                terminal.disposition_detail.as_deref(),
15646                Some(policy.budget_exhausted_detail().as_str()),
15647                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
15648            );
15649            assert_eq!(child.spawn_failure, None);
15650            child.release_roster();
15651        }
15652    }
15653}
15654
15655/// The daemon's real spawn path hands a subc-wire child its launch nonce on
15656/// descriptor 3, without an environment copy. The shell records the nonce
15657/// and its environment after exec so these tests observe the real handover.
15658#[cfg(all(test, unix))]
15659mod launch_nonce_descriptor_tests {
15660    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
15661    use crate::stderr_tail::{StderrRing, StderrTailConfig};
15662    use cortexkit_test_support::ScratchDir;
15663    use std::{
15664        path::PathBuf,
15665        sync::{Arc, Mutex},
15666        time::{Duration, Instant},
15667    };
15668
15669    async fn probe(role: super::SpawnRole) {
15670        let scratch = ScratchDir::new("launch-nonce-descriptor");
15671        let fd_copy = scratch.join("from-descriptor");
15672        let env_copy = scratch.join("environment");
15673        let script = format!(
15674            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
15675            fd = fd_copy.display(), env = env_copy.display(),
15676        );
15677        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
15678        let spec = ModuleSpec {
15679            module_id: "nonce-descriptor-probe".to_string(),
15680            program: PathBuf::from("/bin/sh"),
15681            args: vec!["-c".to_string(), script],
15682            env: vec![
15683                xdg("XDG_DATA_HOME"),
15684                xdg("XDG_RUNTIME_DIR"),
15685                xdg("XDG_CONFIG_HOME"),
15686                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
15687            ],
15688            reserved: true,
15689            reserved_prefixes: Vec::new(),
15690            protocol: ModuleProtocol::Subc,
15691            overlap: Default::default(),
15692        };
15693        let handle = SupervisorHandle::new();
15694        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15695        let roster = ChildRoster::default();
15696        #[cfg(target_os = "macos")]
15697        {
15698            let path = super::test_privacy_trampoline();
15699            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
15700        }
15701        let child = super::spawn_child_in_slot(
15702            &spec,
15703            None,
15704            Some(&handle),
15705            &ring,
15706            None,
15707            &roster,
15708            #[cfg(target_os = "linux")]
15709            None,
15710            role,
15711            matches!(role, super::SpawnRole::SwapCandidate),
15712        )
15713        .expect("spawn probe");
15714        let deadline = Instant::now() + Duration::from_secs(10);
15715        while !(fd_copy.exists() && env_copy.exists()) {
15716            assert!(Instant::now() < deadline, "probe never wrote its copies");
15717            tokio::time::sleep(Duration::from_millis(20)).await;
15718        }
15719        let nonce = std::fs::read_to_string(fd_copy).unwrap();
15720        assert!(!nonce.is_empty());
15721        let environment = std::fs::read_to_string(env_copy).unwrap();
15722        assert!(environment
15723            .lines()
15724            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
15725        let copy = environment
15726            .lines()
15727            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
15728        assert_eq!(
15729            copy, None,
15730            "Unix children must never receive the environment nonce"
15731        );
15732        if matches!(role, super::SpawnRole::Plain) {
15733            assert_eq!(
15734                handle.spawn_nonce(&spec.module_id).as_deref(),
15735                Some(nonce.as_str())
15736            );
15737        }
15738        drop(child);
15739    }
15740
15741    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15742    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
15743        probe(super::SpawnRole::Plain).await;
15744    }
15745
15746    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15747    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
15748        probe(super::SpawnRole::SwapCandidate).await;
15749    }
15750}
15751
15752#[cfg(all(test, target_os = "linux"))]
15753mod cgroup_containment_tests {
15754    use super::*;
15755    use cortexkit_test_support::ScratchDir;
15756
15757    fn running(pid: u32) -> bool {
15758        // An orphan can remain a zombie until the container init reaps it.
15759        std::fs::read_to_string(format!("/proc/{pid}/stat"))
15760            .ok()
15761            .and_then(|stat| {
15762                stat.rsplit_once(") ")
15763                    .map(|(_, rest)| rest.starts_with('Z'))
15764            })
15765            .is_some_and(|zombie| !zombie)
15766    }
15767
15768    #[tokio::test]
15769    async fn linux_teardown_reaps_the_grandchild() {
15770        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
15771    }
15772
15773    #[tokio::test]
15774    async fn linux_shutdown_straggler_reaps_the_grandchild() {
15775        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
15776    }
15777
15778    async fn teardown_tree(test_name: &str, shutdown: bool) {
15779        let dir = ScratchDir::new(test_name);
15780        let root = PathBuf::from(format!(
15781            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
15782            std::process::id(),
15783            unix_ms_now()
15784        ));
15785        if let Err(error) = std::fs::create_dir(&root) {
15786            assert!(
15787                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15788                "required cgroup test cannot execute: {error}"
15789            );
15790            eprintln!(
15791                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
15792                root.display()
15793            );
15794            return;
15795        }
15796        let placement = subc_cgroup::prepare_at(&root)
15797            .expect("prepare isolated kernel cgroup")
15798            .expect("isolated cgroup is delegated");
15799        let module_id = "tree-teardown";
15800        let module = placement
15801            .module_path(module_id)
15802            .expect("create isolated module cgroup");
15803        if !module.join("cgroup.kill").exists() {
15804            std::fs::remove_dir(&module).unwrap();
15805            std::fs::remove_dir(root.join("subc-modules")).unwrap();
15806            std::fs::remove_dir(&root).unwrap();
15807            assert!(
15808                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15809                "required cgroup.kill interface unavailable"
15810            );
15811            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
15812            return;
15813        }
15814        let supervisor = Supervisor::new_for_test(
15815            Arc::new(Registry::default()),
15816            RestartPolicy::new(3, Duration::ZERO),
15817        )
15818        .with_cgroup_placement(Some(placement));
15819        let mut runtime = supervisor.runtime_config();
15820        runtime.child_roster = runtime
15821            .child_roster
15822            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
15823        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15824        let pid_file = dir.join("grandchild.pid");
15825        let spec = ModuleSpec {
15826            module_id: module_id.to_string(),
15827            program: PathBuf::from("/bin/sh"),
15828            args: vec![
15829                "-c".into(),
15830                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
15831                "fixture".into(),
15832                pid_file.display().to_string(),
15833            ],
15834            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15835                .into_iter()
15836                .map(|key| (key.to_string(), dir.display().to_string()))
15837                .collect(),
15838            reserved: false,
15839            reserved_prefixes: Vec::new(),
15840            protocol: ModuleProtocol::None,
15841            overlap: Default::default(),
15842        };
15843        let child =
15844            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
15845        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
15846        let grandchild: u32 = loop {
15847            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
15848                if let Ok(pid) = contents.trim().parse() {
15849                    break pid;
15850                }
15851            }
15852            assert!(
15853                tokio::time::Instant::now() < deadline,
15854                "grandchild pid was not recorded"
15855            );
15856            tokio::time::sleep(Duration::from_millis(10)).await;
15857        };
15858        assert!(
15859            running(grandchild),
15860            "grandchild must be alive before teardown"
15861        );
15862        if shutdown {
15863            let mut child = child;
15864            crate::child_roster::end_children_for_daemon_shutdown(
15865                &runtime.child_roster,
15866                false,
15867                std::future::pending(),
15868            )
15869            .await;
15870            child.wait().await.expect("reap shutdown straggler");
15871        } else {
15872            drain_child_to_state(
15873                module_id,
15874                ModuleProtocol::None,
15875                StopNotice::NotSent,
15876                &Registry::default(),
15877                None,
15878                &snapshot,
15879                &runtime.terminal_ring,
15880                &SpawnEventFeed::default(),
15881                child,
15882                Duration::from_millis(100),
15883                ModuleState::Stopped,
15884                Some(false),
15885            )
15886            .await
15887            .expect("real supervisor teardown");
15888        }
15889        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
15890        while running(grandchild) && tokio::time::Instant::now() < deadline {
15891            tokio::time::sleep(Duration::from_millis(10)).await;
15892        }
15893        let survived = running(grandchild);
15894        // Kill a surviving grandchild so a failed test does not leave it behind.
15895        if survived {
15896            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
15897            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
15898            tokio::time::sleep(Duration::from_millis(100)).await;
15899        }
15900        if module.exists() {
15901            std::fs::remove_dir(&module).expect("remove empty module cgroup");
15902        }
15903        std::fs::remove_dir(root.join("subc-modules")).unwrap();
15904        std::fs::remove_dir(&root).unwrap();
15905        assert!(
15906            !survived,
15907            "grandchild {grandchild} outlived module teardown"
15908        );
15909        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
15910    }
15911}