Skip to main content

subc_daemon/
supervise.rs

1use std::{
2    collections::{HashMap, HashSet, VecDeque},
3    error::Error,
4    fmt, io,
5    path::PathBuf,
6    process::{ExitStatus, Stdio},
7    sync::{Arc, Mutex, OnceLock},
8    time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19    manifest::{SelfSignalKind, SignalAnchor},
20    session::{
21        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22        MODULE_CONTROL_OP_HEALTH_CHECK,
23    },
24    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27    process::{Child, Command},
28    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29    task::JoinHandle,
30    time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35    child_roster::ChildRoster,
36    daemon_config::{
37        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38    },
39    forwarding::{
40        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41        ModuleDrainTarget, PendingModuleControlRpc,
42    },
43    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44    registry::{ConnectionId, RegistrationEndReason, RegistryError},
45    stderr_tail::{
46        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47        StderrTailSnapshot,
48    },
49    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50    Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62/// Command-line flag used by supervised modules to find subc.
63///
64/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
65/// The path points at the TCP+key connection file; it is not an ambient signal and
66/// is never inherited by standalone children.
67pub const SUBC_ARG: &str = "--subc";
68
69const DEFAULT_MAX_RESTARTS: u32 = 3;
70const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
71const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
72/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
73/// to contain a real crash loop (which respawns in seconds) and short enough
74/// that unrelated crashes hours apart never accumulate into a permanent stop.
75const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
76/// How long a drain waits for already-dispatched requests to finalize before
77/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
78/// not RPC-scale: the original 2s value silently cut nearly every real tool
79/// call at the fence, making the wait-for-finalize design decorative for the
80/// workloads it existed for. Quiescence short-circuits, so an idle module
81/// restarts immediately regardless of this value; the budget is spent only
82/// when a genuine in-flight request is worth finishing. Per-module override:
83/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
84/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
85/// where a stuck request will never settle).
86pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
87const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
88const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
89/// How long a drain waits after one busy-gauge probe completes before it asks
90/// again.
91///
92/// A drain re-probes on this short interval of its own rather than on the
93/// module's health cadence. The default cadence (30 s) equals the default
94/// drain ceiling, so a cadence-paced drain that found the module busy at its
95/// first probe never asked a second time: it waited out the whole ceiling and
96/// forced GOODBYE even after the gauges had read zero. The interval is counted
97/// from the moment the previous probe finished, and probes run one at a time,
98/// so a slow module is never asked more often than it can answer. Periodic
99/// health checks outside a drain keep using the configured cadence.
100const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
101/// How long a restart waits for the exited process's output readers.
102///
103/// The restart does not depend on either output reader finishing. A stderr
104/// reader still running at this bound is left running, and its later output goes
105/// into the exited process's own section of the stderr ring (see
106/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
107/// bound no longer decides whether a crash's last lines are kept: under load
108/// the reader may simply not have been scheduled yet, and cutting it there
109/// discarded exactly the lines that explained the crash.
110///
111/// What the bound still decides is when the tail starts reporting
112/// `Incomplete`: a pipe open past it usually means a descendant of the exited
113/// process still holds it, and the tail cannot claim to be whole until that
114/// pipe closes. That is also the only case in which the wait costs the restart
115/// anything, because a pipe with no other holder reaches EOF when the process
116/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
117/// `Captured` at EOF with nothing lost.
118///
119/// The stdout reader carries no ring, only the capture file. It also continues
120/// to EOF after this bound: child exit closes the writer but does not mean the
121/// reader has consumed all buffered bytes. Late output from either pipe can
122/// therefore reach the shared capture file after the successor starts.
123const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
124/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
125pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
126const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
127/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
128/// dropped because the subscriber stopped draining its frames. The detail's
129/// `first_undelivered_cursor` names the first event it did not receive; the
130/// client resubscribes from the last cursor it did receive.
131pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
132
133#[cfg(target_os = "macos")]
134struct PrivacyExec {
135    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
136    deadline: tokio::time::Instant,
137    expected: Option<subc_os::FileIdentity>,
138    trampoline: Option<subc_os::FileIdentity>,
139    script: bool,
140    module_id: String,
141    #[cfg(test)]
142    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
143}
144
145#[cfg(all(test, target_os = "macos"))]
146pub(crate) fn test_privacy_trampoline() -> PathBuf {
147    // Cargo's unit-test executable lives in <profile>/deps; its fixture bin
148    // lives beside that directory. This honors custom CARGO_TARGET_DIR too.
149    let path = std::env::current_exe()
150        .unwrap()
151        .parent()
152        .unwrap()
153        .parent()
154        .unwrap()
155        .join("privacy-trampoline-fixture");
156    // Without the fixture every macOS spawn is refused, and the tests that
157    // spawn fail later as a module in state Failed, which names the wrong
158    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
159    assert!(
160        path.exists(),
161        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
162         --bins --features test-support` or `cargo test -p subc-daemon` first",
163        path.display()
164    );
165    path
166}
167
168#[cfg(target_os = "macos")]
169fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
170    use std::io::Read;
171    let mut probe = std::process::Command::new(path)
172        .args(["__disclaim-exec", "--probe"])
173        .stdin(Stdio::null())
174        .stdout(Stdio::piped())
175        .stderr(Stdio::piped())
176        .spawn()
177        .map_err(|error| {
178            format!(
179                "privacy trampoline probe failed for {}: {error}",
180                path.display()
181            )
182        })?;
183    let deadline = std::time::Instant::now() + Duration::from_secs(5);
184    let status = loop {
185        match probe.try_wait() {
186            Ok(Some(status)) => break status,
187            Ok(None) if std::time::Instant::now() < deadline => {
188                std::thread::sleep(Duration::from_millis(5))
189            }
190            result => {
191                let _ = probe.kill();
192                let _ = probe.wait();
193                return Err(format!(
194                    "privacy trampoline probe failed or timed out for {}: {result:?}",
195                    path.display()
196                ));
197            }
198        }
199    };
200    let mut answer = String::new();
201    if let Some(stdout) = probe.stdout.take() {
202        let _ = stdout.take(256).read_to_string(&mut answer);
203    }
204    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
205        return Ok(());
206    }
207    let mut diagnostic = String::new();
208    if let Some(stderr) = probe.stderr.take() {
209        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
210    }
211    let cause = diagnostic
212        .trim()
213        .strip_prefix("ck-subc: own privacy identity refused: ")
214        .unwrap_or("binary does not implement the privacy trampoline protocol");
215    Err(format!(
216        "{cause}: probe of {} exited {status}",
217        path.display()
218    ))
219}
220
221#[cfg(target_os = "macos")]
222fn privacy_command(
223    spec: &ModuleSpec,
224    roster: &ChildRoster,
225) -> Result<
226    (
227        Command,
228        Option<PrivacyExec>,
229        subc_os::privacy_identity::ExecAcknowledgement,
230    ),
231    SuperviseError,
232> {
233    let failure = |cause: String| {
234        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
235        SuperviseError::Spawn {
236            program: spec.program.clone(),
237            source: io::Error::other(cause),
238            cgroup_path: None,
239        }
240    };
241    let trampoline = roster.privacy_trampoline().map_err(failure)?;
242    // Resolve PATH with the same environment the Command will receive. For
243    // scripts retain the existing orphan-identity rule: the kernel chooses
244    // the interpreter, and its observed image is the one recorded. Do not
245    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
246    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
247        let path = spec
248            .env
249            .iter()
250            .find(|(key, _)| key == "PATH")
251            .map(|(_, value)| std::ffi::OsString::from(value))
252            .or_else(|| std::env::var_os("PATH"))
253            .unwrap_or_else(|| "/usr/bin:/bin".into());
254        std::env::split_paths(&path)
255            .map(|dir| dir.join(&spec.program))
256            .find(|path| path.is_file())
257            .unwrap_or_else(|| spec.program.clone())
258    } else {
259        spec.program.clone()
260    };
261    let expected = subc_os::file_identity(&program);
262    let trampoline_image = subc_os::file_identity(&trampoline);
263    let script = {
264        use std::io::Read;
265        let mut prefix = [0u8; 2];
266        std::fs::File::open(&program)
267            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
268    };
269    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
270        return Err(failure(
271            "privacy identity module executable is missing or is the trampoline itself".to_string(),
272        ));
273    }
274    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
275        .map_err(|error| failure(error.to_string()))?;
276    let reader =
277        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
278    let mut command = Command::new(&trampoline);
279    command
280        .arg("__disclaim-exec")
281        .arg(ack.fd().to_string())
282        .arg(&program);
283    ack.install(command.as_std_mut());
284    Ok((
285        command,
286        Some(PrivacyExec {
287            reader,
288            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
289            expected,
290            trampoline: trampoline_image,
291            script,
292            module_id: spec.module_id.clone(),
293            #[cfg(test)]
294            confirmation_pause: spec
295                .env
296                .iter()
297                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
298                .map(|(_, address)| (address.clone(), None)),
299        }),
300        ack,
301    ))
302}
303
304struct SupervisedChild {
305    child: Child,
306    exec: crate::activity::ExecOwner,
307    #[cfg(windows)]
308    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
309    #[cfg(target_os = "macos")]
310    privacy_exec: Option<PrivacyExec>,
311    /// Set once this launch's exec acknowledgement confirms the module image.
312    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
313    /// `subc_os::privacy_identity`), which then replaces itself with the
314    /// module. The supervisor owns and can kill that pid from spawn, but
315    /// status readers report it only after this latch is set, so nothing
316    /// reports the trampoline's image as the module's.
317    #[cfg(target_os = "macos")]
318    report_ready: Arc<OnceLock<()>>,
319    /// Refusal before the module image was accepted, retained for terminal records.
320    spawn_failure: Option<String>,
321    /// The protocol this process was launched with. A reload can store a new
322    /// launch spec with a different protocol, but that takes effect only at the
323    /// next spawn, so this process keeps being handled by the protocol it
324    /// actually speaks.
325    protocol: ModuleProtocol,
326    /// This process's cgroup name: a bounded module/slot label followed by a
327    /// spawn suffix unique to this process (when cgroup placement is on). A
328    /// retired process in a slot may still be draining when a later one is
329    /// spawned into that slot, so the suffix keeps the later process out of
330    /// the retired one's cgroup, which is the domain a kill applies to.
331    #[cfg(target_os = "linux")]
332    module_id: String,
333    #[cfg(target_os = "linux")]
334    cgroup_placement: Option<subc_cgroup::Placement>,
335    /// The job that contains this child and every process it spawns (issue #109).
336    ///
337    /// Dropping this handle is what reaps a surviving tree when no supervisor
338    /// code runs — a daemon crash — because the job carries
339    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
340    ///
341    /// That limit is not crash-only, and the difference is worth knowing: a
342    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
343    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
344    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
345    /// module at once. Before this change they survived that, saw EOF on the
346    /// control socket, and ran their own teardown; Unix keeps that path
347    /// deliberately, so a module can seal a WAL or close a capture rather than
348    /// be killed mid-write. So this trades graceful teardown on every Windows
349    /// daemon stop for containment on a crash, which is the right way round
350    /// today: orphaned GPU workers are a reported, recurring problem, and the
351    /// modules that write most heavily do not run on Windows.
352    ///
353    /// The fix is a real Windows stop path — the daemon draining before it
354    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
355    /// reaches only what the drain left behind, which is what it should reach.
356    #[cfg(windows)]
357    job: Option<subc_jobobject::JobObject>,
358    stdout_pump: Option<JoinHandle<()>>,
359    stderr_pump: Option<StderrPump>,
360    stderr_ring: Arc<Mutex<StderrRing>>,
361    spawned_at_ms: u64,
362    spawned_from: PathBuf,
363    launch_spec_digest: LaunchSpecDigest,
364    spawned_file_identity: Option<SpawnedFileIdentity>,
365    process_start_time: Option<u64>,
366    process_identity: Option<ProcessIdentity>,
367    pid: u32,
368    /// This process's entry in the daemon's child roster, released when the
369    /// process is reaped or this handle is dropped.
370    roster_guard: Option<crate::child_roster::RosterGuard>,
371}
372
373impl SupervisedChild {
374    fn id(&self) -> Option<u32> {
375        Some(self.pid)
376    }
377
378    fn process_identity(&self) -> Option<ProcessIdentity> {
379        self.process_identity
380    }
381
382    async fn wait(&mut self) -> io::Result<ExitStatus> {
383        #[cfg(target_os = "macos")]
384        self.confirm_privacy_exec().await;
385        // The roster entry is NOT released here. A daemon shutdown waits for the
386        // roster to empty and then exits the process, so releasing at the reap
387        // let it exit before the exit handler wrote this child's terminal record
388        // (the stderr drain and snapshot update sit in between), and the
389        // shutdown's own `daemon_shutdown` record was intermittently lost. The
390        // caller releases it after recording the exit (`release_roster`), and
391        // dropping the handle releases it too.
392        let result = self.child.wait().await;
393        #[cfg(target_os = "linux")]
394        if result.is_ok() {
395            if let Some(placement) = self.cgroup_placement.as_ref() {
396                cleanup_reaped_cgroup(placement, &self.module_id).await;
397                // Keep ownership while awaiting kernel population changes: a
398                // drain timeout may cancel this wait and then escalate/reap.
399                self.cgroup_placement = None;
400            }
401        }
402        result
403    }
404
405    #[cfg(target_os = "macos")]
406    async fn confirm_privacy_exec(&mut self) {
407        let Some(pending) = &mut self.privacy_exec else {
408            return;
409        };
410        let result = tokio::time::timeout_at(pending.deadline, async {
411            let mut record = Vec::new();
412            loop {
413                let mut ready = pending.reader.readable().await?;
414                let read = ready.try_io(|reader| {
415                    use std::io::Read;
416                    let mut reader = reader.get_ref();
417                    let mut buffer = [0u8; 256];
418                    reader.read(&mut buffer).map(|count| (count, buffer))
419                });
420                match read {
421                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
422                    Ok(Ok((count, buffer))) => {
423                        if record.len() + count > 1024 {
424                            return Err(io::Error::other(
425                                "privacy exec refusal record is too long",
426                            ));
427                        }
428                        record.extend_from_slice(&buffer[..count]);
429                    }
430                    Ok(Err(error)) => return Err(error),
431                    Err(_) => continue,
432                }
433            }
434        })
435        .await;
436        #[cfg(test)]
437        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
438            use tokio::io::{AsyncReadExt, AsyncWriteExt};
439            if stream.is_none() {
440                *stream = Some(
441                    tokio::net::TcpStream::connect(address.as_str())
442                        .await
443                        .unwrap(),
444                );
445                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
446                stream
447                    .as_mut()
448                    .unwrap()
449                    .write_all(&self.pid.to_ne_bytes())
450                    .await
451                    .unwrap();
452            }
453            let mut reply = [0];
454            stream
455                .as_mut()
456                .unwrap()
457                .read_exact(&mut reply)
458                .await
459                .unwrap();
460            pending.confirmation_pause = None;
461            if &reply == b"F" {
462                Ok(Err(io::Error::other("injected late exec refusal")))
463            } else {
464                assert_eq!(&reply, b"X");
465                result
466            }
467        } else {
468            result
469        };
470        // Keep the reader in self across await: select cancellation must not
471        // discard the handshake or reset its original five-second deadline.
472        let pending = self.privacy_exec.as_ref().expect("pending exec");
473        let cause = match result {
474            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
475            Ok(Err(error)) => Some(format!(
476                "privacy identity exec acknowledgement failed: {error}"
477            )),
478            Ok(Ok(record)) if !record.is_empty() => Some(
479                std::str::from_utf8(&record)
480                    .ok()
481                    .and_then(|record| {
482                        record
483                            .trim()
484                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
485                    })
486                    .filter(|cause| !cause.is_empty())
487                    .unwrap_or("invalid privacy exec refusal record")
488                    .to_string(),
489            ),
490            Ok(Ok(_)) => match self.child.try_wait() {
491                // Empty EOF is the exec acknowledgement. A real module may exit
492                // immediately, including with a reserved trampoline status; no
493                // image is admitted, and its ordinary exit contract stays intact.
494                Ok(Some(_status)) => None,
495                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
496                Ok(None) => {
497                    let image = observe_spawned_image(self.pid);
498                    if let Some(image) = image.filter(|image| {
499                        image.executable.is_some()
500                            && image.executable != pending.trampoline
501                            && (image.executable == pending.expected || pending.script)
502                    }) {
503                        if let Err(error) = self
504                            .roster_guard
505                            .as_ref()
506                            .map(|guard| guard.confirm_executable(image))
507                            .transpose()
508                        {
509                            Some(format!(
510                                "could not durably record confirmed module image: {error}"
511                            ))
512                        } else {
513                            let _ = self.report_ready.set(());
514                            info!(module_id = %pending.module_id, pid = self.pid,
515                                "module spawned with own privacy identity (responsibility disclaimed)");
516                            None
517                        }
518                    } else if image.is_none()
519                        || image.is_some_and(|image| image.executable.is_none())
520                    {
521                        // A process can exit between try_wait and the kernel
522                        // image read. Empty EOF already acknowledged exec, so
523                        // preserve that module's ordinary exit rather than
524                        // mislabel a disappearing image as trampoline refusal.
525                        // Keep pending in self across await so cancellation does
526                        // not discard validation or reset its original deadline.
527                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
528                            Ok(Ok(_status)) => None,
529                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
530                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
531                        }
532                    } else {
533                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
534                    }
535                }
536            },
537        };
538        let pending = self.privacy_exec.take().expect("pending exec");
539        // Exec confirmation describes launch success, not roster retirement.
540        // Keep even a reaped child recorded until its terminal history exists.
541        self.exec.0.finish(if cause.is_some() {
542            crate::activity::ExecState::Refused
543        } else {
544            crate::activity::ExecState::CommittedExec
545        });
546        if let Some(cause) = cause {
547            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
548            self.spawn_failure = Some(cause);
549            // No image is admitted on failure. Reach the entire fresh process
550            // group, including a module which spawned a helper before refusal.
551            if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
552                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
553            }
554            let _ = self.child.start_kill();
555        }
556    }
557
558    /// Releases this child's daemon-shutdown roster entry once its exit has
559    /// been recorded. The pid is already reaped and free for reuse, so the
560    /// entry must not outlive the record any longer than that.
561    fn release_roster(&mut self) {
562        self.roster_guard = None;
563    }
564
565    /// Kill the child and, where containment is available, its process tree.
566    ///
567    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
568    /// helper process leaked the helper — the Synapse embedding module's CUDA
569    /// worker holds the GPU allocation, so the leak cost VRAM until the next
570    /// restart of something else. Terminating the job reaches grandchildren that
571    /// a tree walk cannot, including one whose parent has already exited and
572    /// been reparented away.
573    ///
574    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
575    /// direct-child kill still decides the outcome, so containment can never
576    /// change whether a module is reported as stopped.
577    fn start_kill(&mut self) -> io::Result<()> {
578        #[cfg(windows)]
579        if let Some(job) = &self.job {
580            if let Err(error) = job.terminate() {
581                debug!(
582                    error = %error,
583                    "job termination failed; the direct-child kill still owns the outcome"
584                );
585            }
586        }
587        #[cfg(target_os = "linux")]
588        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
589        self.child.start_kill()
590    }
591
592    async fn drain_stderr(&mut self, module_id: &str) {
593        if let Some(pump) = self.stdout_pump.take() {
594            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
595        }
596
597        let Some(pump) = self.stderr_pump.take() else {
598            return;
599        };
600        settle_stderr_pump(
601            module_id,
602            &self.stderr_ring,
603            pump,
604            STDERR_PUMP_DRAIN_TIMEOUT,
605        )
606        .await;
607    }
608}
609
610/// Bound the restart's wait, not the lifetime of the exited child's stdout
611/// reader. Slow pipe reads or capture writes must not turn the bound into lost
612/// output; dropping the handle lets the reader finish at EOF, like stderr.
613async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
614    match timeout(bound, &mut pump).await {
615        Ok(Ok(())) => {}
616        Ok(Err(error)) => {
617            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
618        }
619        Err(_) => {
620            drop(pump);
621            warn!(
622                module_id,
623                waited = ?bound,
624                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
625            );
626        }
627    }
628}
629
630/// The reader task for one process's stderr, with the ring generation its
631/// lines are attributed to.
632struct StderrPump {
633    task: JoinHandle<()>,
634    generation: u64,
635}
636
637/// Retire an exited process's stderr reader and wait up to `bound` for it to
638/// reach EOF. A reader still running at the bound is detached, not stopped: it
639/// keeps filling the exited process's section of the ring until its pipe
640/// closes, and the tail reads `Incomplete` until then. See
641/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
642async fn settle_stderr_pump(
643    module_id: &str,
644    ring: &Arc<Mutex<StderrRing>>,
645    pump: StderrPump,
646    bound: Duration,
647) {
648    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
649    let StderrPump {
650        mut task,
651        generation,
652    } = pump;
653    lock().retire_pump(generation);
654    match timeout(bound, &mut task).await {
655        Ok(Ok(())) => {}
656        Ok(Err(err)) => {
657            let mut ring = lock();
658            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
659            ring.finish_pump(generation);
660            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
661        }
662        Err(_) => {
663            // Dropping the handle detaches the task; it ends at EOF on its pipe.
664            drop(task);
665            lock().mark_pump_late(
666                generation,
667                format!(
668                    "stderr of the exited process had not reached EOF {bound:?} after it was \
669                     retired (a descendant may still hold the pipe open); lines it still \
670                     writes are kept in that process's section"
671                ),
672            );
673            warn!(
674                module_id,
675                waited = ?bound,
676                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
677            );
678        }
679    }
680}
681
682fn registration_release_events() -> &'static watch::Sender<u64> {
683    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
684    EVENTS.get_or_init(|| {
685        let (sender, _receiver) = watch::channel(0);
686        sender
687    })
688}
689
690pub(crate) fn notify_registration_release() {
691    let events = registration_release_events();
692    let next_generation = (*events.borrow()).wrapping_add(1);
693    events.send_replace(next_generation);
694}
695
696/// How to launch one singleton module process.
697#[derive(Debug, Clone, PartialEq, Eq)]
698pub struct ModuleSpec {
699    pub module_id: String,
700    pub program: PathBuf,
701    pub args: Vec<String>,
702    pub env: Vec<(String, String)>,
703    /// When true this is a reserved module: each spawn gets a fresh one-time launch
704    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
705    /// process can register this module_id (a security-boundary module like the
706    /// credential vault must not be impersonable while it is down/restarting).
707    pub reserved: bool,
708    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
709    /// Prefixes come from daemon config and must end in `:` before they reach the
710    /// supervisor; the owner module's current spawn nonce authorizes claims under
711    /// each prefix.
712    pub reserved_prefixes: Vec<String>,
713    /// The wire protocol this module speaks, as DECLARED in daemon config.
714    ///
715    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
716    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
717    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
718    /// and NO launch nonce, and a clean exit the daemon did not request is
719    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
720    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
721    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
722    /// because a process ignores an environment variable it does not read.
723    ///
724    /// The argument is the part that cannot be "harmless to a process that
725    /// ignores it": a stock binary exits on an unknown flag before it listens
726    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
727    /// first conformance run against this mode found it. The nonce is withheld
728    /// because a process that will never present it gains nothing from holding
729    /// it, and a secret in the environment of a process that does not need it is
730    /// a leak surface for no benefit.
731    pub protocol: ModuleProtocol,
732    /// Whether two processes of this module may run at once, which is what a
733    /// blue/green swap does for the length of its overlap. Declared in daemon
734    /// config because the daemon must be able to answer it while the module is
735    /// down, and so a module cannot talk itself into it after registering.
736    pub overlap: ModuleOverlap,
737}
738
739/// Only the configured launch inputs enter this value; spawned connection paths,
740/// nonces and inherited environment never do. Its bytes remain daemon-private.
741#[derive(Clone, PartialEq, Eq)]
742pub(crate) struct LaunchSpecDigest([u8; 32]);
743
744impl fmt::Debug for LaunchSpecDigest {
745    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
746        f.write_str("LaunchSpecDigest([redacted])")
747    }
748}
749
750impl LaunchSpecDigest {
751    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
752        fn field(hash: &mut Sha256, bytes: &[u8]) {
753            hash.update((bytes.len() as u64).to_le_bytes());
754            hash.update(bytes);
755        }
756        let mut hash = Sha256::new();
757        if let Some(program) = spec.program.to_str() {
758            field(&mut hash, b"utf8-program");
759            field(&mut hash, program.as_bytes());
760        } else {
761            // The daemon config is UTF-8, but in-process callers may provide
762            // native paths: lossy conversion would collapse distinct programs.
763            #[cfg(unix)]
764            {
765                use std::os::unix::ffi::OsStrExt;
766                field(&mut hash, b"unix-program");
767                field(&mut hash, spec.program.as_os_str().as_bytes());
768            }
769            #[cfg(windows)]
770            {
771                use std::os::windows::ffi::OsStrExt;
772                field(&mut hash, b"windows-program");
773                let bytes = spec
774                    .program
775                    .as_os_str()
776                    .encode_wide()
777                    .flat_map(u16::to_le_bytes)
778                    .collect::<Vec<_>>();
779                field(&mut hash, &bytes);
780            }
781            #[cfg(not(any(unix, windows)))]
782            field(&mut hash, spec.program.to_string_lossy().as_bytes());
783        }
784        hash.update((spec.args.len() as u64).to_le_bytes());
785        for arg in &spec.args {
786            field(&mut hash, arg.as_bytes());
787        }
788        let mut env = spec.env.iter().collect::<Vec<_>>();
789        env.sort_unstable();
790        hash.update((env.len() as u64).to_le_bytes());
791        for (key, value) in env {
792            field(&mut hash, key.as_bytes());
793            field(&mut hash, value.as_bytes());
794        }
795        field(
796            &mut hash,
797            match spec.protocol {
798                ModuleProtocol::Subc => b"subc",
799                ModuleProtocol::None => b"none",
800            },
801        );
802        Self(hash.finalize().into())
803    }
804}
805
806/// Whether a module tolerates a second process of itself running alongside.
807///
808/// Most modules are single-writer on their store (a WAL, a capture log, a
809/// resident index behind a writer barrier), and two processes on one store
810/// corrupt it. So a swap, which overlaps the old and new process by design,
811/// is refused unless the module's config opts in with `overlap: "safe"`.
812#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
813pub enum ModuleOverlap {
814    /// Never run two processes of this module at once. The default.
815    #[default]
816    Exclusive,
817    /// The module has said a second process of itself is harmless for the
818    /// length of a swap.
819    ///
820    /// Declare it only if a second instance can run for a few seconds without
821    /// touching ANY single-writer store: every database, WAL, index, projector
822    /// and scheduled job the module owns. A lease on part of that state is not
823    /// enough. broca's session lease guards WAL appends while its run index, its
824    /// store projector and its archive fold timer (which unlinks live WAL files)
825    /// stay single-writer, so broca is exclusive despite holding a lease. The
826    /// refusal only fires after this has been decided, so the decision is the
827    /// check.
828    Safe,
829}
830
831impl ModuleOverlap {
832    pub fn as_str(self) -> &'static str {
833        match self {
834            Self::Exclusive => "exclusive",
835            Self::Safe => "safe",
836        }
837    }
838}
839
840/// Environment variable telling a spawned module which case it was started
841/// for, before it sends HELLO. Only a swap candidate carries it, as
842/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
843///
844/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
845/// longer because nobody waits on it, while a plain restart must flip ready
846/// quickly because callers see `module_warming` until it does. Absence means
847/// plain restart, the safe reading. The daemon trusts nothing about it; the
848/// candidate is proven by its launch nonce at HELLO.
849pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
850/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
851pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
852/// How long a swap waits for its candidate to register and declare itself
853/// ready when the operator does not say. A module warming as a swap candidate
854/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
855/// daemon allows that plus time to start the process and send HELLO.
856pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
857
858/// Bounded restart policy for crash exits.
859///
860/// `max_restarts` is the number of replacement processes allowed after the
861/// initial spawn WITHIN `window`. After that many crash restarts inside one
862/// window the module enters [`ModuleState::Failed`] and the supervisor stops
863/// the crash loop.
864///
865/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
866/// and that only survived because crashes were rare: a module that crashed
867/// three times across a week was disabled forever by crashes that had nothing
868/// to do with each other. That stopped being survivable once modules began
869/// exiting non-zero whenever the daemon's connection to them drops, because
870/// then every daemon-side connection drop spends a unit of the same budget and
871/// one flappy hour permanently stops a healthy module. Restarts older than
872/// `window` release their slot, so a module that crashed twice yesterday has a
873/// full budget today, while a genuine crash loop -- which is fast by
874/// definition -- still reaches the cap and stops.
875#[derive(Debug, Clone, Copy, PartialEq, Eq)]
876pub struct RestartPolicy {
877    pub max_restarts: u32,
878    /// Base delay before a crash replacement. The actual delay escalates with
879    /// the number of recent crash replacements and is capped by `max_backoff`.
880    pub backoff: Duration,
881    /// Maximum delay before a crash replacement.
882    pub max_backoff: Duration,
883    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
884    /// budget effectively infinite (nothing is ever in-window), which is why
885    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
886    pub window: Duration,
887}
888
889impl RestartPolicy {
890    /// A policy with the default crash window. Callers that care about the
891    /// window say so with [`Self::with_window`]; the ones that do not are
892    /// asking for the standard rate limit, not for no limit.
893    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
894        Self {
895            max_restarts,
896            backoff,
897            max_backoff: DEFAULT_MAX_BACKOFF,
898            window: DEFAULT_RESTART_WINDOW,
899        }
900    }
901
902    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
903        self.max_backoff = max_backoff;
904        self
905    }
906
907    pub fn with_window(mut self, window: Duration) -> Self {
908        self.window = window;
909        self
910    }
911
912    /// Calculate the capped exponential delay for the next crash replacement.
913    /// `restart_in_window` is zero for the first replacement after an operator
914    /// action (restart, reload, re-enable) cleared the crash ring, or after all
915    /// older crash replacements have aged out of the window.
916    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
917        if self.backoff.is_zero() || self.max_backoff.is_zero() {
918            return Duration::ZERO;
919        }
920
921        let mut delay = self.backoff;
922        for _ in 0..restart_in_window {
923            if delay >= self.max_backoff {
924                return self.max_backoff;
925            }
926            delay = delay
927                .checked_mul(10)
928                .unwrap_or(self.max_backoff)
929                .min(self.max_backoff);
930        }
931        delay.min(self.max_backoff)
932    }
933
934    /// The one sentence that explains a budget-exhausted stop, used for both the
935    /// log line and the terminal record so the two cannot drift. It names the
936    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
937    /// exactly what this budget is not.
938    fn budget_exhausted_detail(&self) -> String {
939        format!(
940            "crash budget exhausted: max_restarts={} within window_secs={}",
941            self.max_restarts,
942            self.window.as_secs()
943        )
944    }
945}
946
947impl Default for RestartPolicy {
948    fn default() -> Self {
949        Self {
950            max_restarts: DEFAULT_MAX_RESTARTS,
951            backoff: DEFAULT_BACKOFF,
952            max_backoff: DEFAULT_MAX_BACKOFF,
953            window: DEFAULT_RESTART_WINDOW,
954        }
955    }
956}
957
958#[derive(Debug, Clone, Copy, PartialEq, Eq)]
959struct CrashRestartSchedule {
960    restart_in_window: u32,
961    delay: Duration,
962}
963
964/// Whether the daemon itself will bring this module back after the exit being
965/// handled: it is enabled AND its in-window crash restarts are below the cap.
966///
967/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
968/// the window are dropped here rather than by a timer, so the count is right
969/// the moment somebody asks and no bookkeeping runs for idle modules.
970fn daemon_will_restart(
971    state: &mut SupervisorSnapshot,
972    policy: &RestartPolicy,
973    now: Instant,
974) -> bool {
975    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
976}
977
978const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
979const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
980const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
981const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
982
983#[derive(Debug, Clone, Copy, PartialEq, Eq)]
984pub enum HealthAction {
985    Report,
986    Restart,
987    Alert,
988}
989
990impl fmt::Display for HealthAction {
991    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
992        f.write_str(match self {
993            Self::Report => "report",
994            Self::Restart => "restart",
995            Self::Alert => "alert",
996        })
997    }
998}
999
1000#[derive(Debug, Clone, PartialEq, Eq)]
1001pub struct HealthConfig {
1002    /// Optional loopback HTTP endpoint for a managed non-wire process.
1003    /// Changing it applies live on rescan; the process protocol changes only
1004    /// at its next spawn.
1005    pub http: Option<String>,
1006    pub cadence: Duration,
1007    pub deadline: Duration,
1008    pub failure_threshold: u32,
1009    pub on_degraded: HealthAction,
1010    pub on_failing: HealthAction,
1011    pub critical: bool,
1012}
1013
1014impl Default for HealthConfig {
1015    fn default() -> Self {
1016        Self {
1017            http: None,
1018            cadence: DEFAULT_HEALTH_CADENCE,
1019            deadline: DEFAULT_HEALTH_DEADLINE,
1020            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1021            on_degraded: HealthAction::Report,
1022            on_failing: HealthAction::Report,
1023            critical: false,
1024        }
1025    }
1026}
1027
1028/// The supervisor's view of one module's health, relayed to clients over
1029/// channel-0 and rendered by `ck health`.
1030///
1031/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1032/// stated here rather than only at the wire type a consumer reads. A reader can
1033/// look up what `None` means; only a writer can silently change it, and the
1034/// writer has no reason to go looking at a downstream contract before editing.
1035///
1036/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1037/// back to `None` on re-registration precisely so a respawned module does not
1038/// carry its predecessor's timestamp — so an old value and an absent one call for
1039/// opposite readings, and anything that defaulted this to a number would make a
1040/// never-probed module indistinguishable from one probed at the epoch.
1041///
1042/// `detail` and `metrics` are `None` when the module published none on this
1043/// probe, which does not mean it reported nothing wrong — it is also the shape
1044/// when the probe never reached it. `last_probe_ms` is what separates those.
1045#[derive(Debug, Clone, PartialEq)]
1046pub struct ModuleHealthStatus {
1047    pub status: SupervisorHealthStatus,
1048    pub last_probe_ms: Option<u64>,
1049    pub detail: Option<String>,
1050    pub metrics: Option<Value>,
1051    pub consecutive_failures: u32,
1052    /// Number of replies received after a recurring health probe's deadline.
1053    /// Unlike a timeout, every increment proves the module was alive.
1054    pub late_answer_count: u64,
1055    /// End-to-end latency of the newest late reply, measured from probe start.
1056    pub last_late_answer_latency_ms: Option<u64>,
1057    pub last_action: Option<String>,
1058    /// Set together with `last_action`; the pair moves as one, and both being
1059    /// absent means no escalation has ever been taken rather than that the last
1060    /// one succeeded.
1061    pub last_action_ms: Option<u64>,
1062}
1063
1064impl Default for ModuleHealthStatus {
1065    fn default() -> Self {
1066        Self {
1067            status: SupervisorHealthStatus::Unknown,
1068            last_probe_ms: None,
1069            detail: None,
1070            metrics: None,
1071            consecutive_failures: 0,
1072            late_answer_count: 0,
1073            last_late_answer_latency_ms: None,
1074            last_action: None,
1075            last_action_ms: None,
1076        }
1077    }
1078}
1079
1080/// Typed lifecycle state for a supervised module.
1081#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1082pub enum ModuleState {
1083    Starting,
1084    Running,
1085    Unresponsive,
1086    Restarting,
1087    Draining,
1088    Stopped,
1089    Failed,
1090    Disabled,
1091}
1092
1093impl fmt::Display for ModuleState {
1094    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1095        f.write_str(match self {
1096            Self::Starting => "starting",
1097            Self::Running => "running",
1098            Self::Unresponsive => "unresponsive",
1099            Self::Restarting => "restarting",
1100            Self::Draining => "draining",
1101            Self::Stopped => "stopped",
1102            Self::Failed => "failed",
1103            Self::Disabled => "disabled",
1104        })
1105    }
1106}
1107
1108/// Supervisor classification of a child-process exit.
1109#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1110pub enum ExitKind {
1111    Clean,
1112    Crash,
1113    DeliberateSeverance,
1114}
1115
1116impl From<ExitKind> for TerminalExitKind {
1117    fn from(kind: ExitKind) -> Self {
1118        match kind {
1119            ExitKind::Clean => Self::Clean,
1120            ExitKind::Crash => Self::Crash,
1121            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1122        }
1123    }
1124}
1125
1126/// Exact process identity retained when a supervised module registers its
1127/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1128#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1129pub(crate) struct ProcessIdentity {
1130    pub(crate) pid: u32,
1131    pub(crate) start_time: u64,
1132}
1133
1134/// Last observed child exit, if any.
1135#[derive(Debug, Clone, PartialEq, Eq)]
1136pub struct ExitReport {
1137    pub kind: ExitKind,
1138    pub code: Option<i32>,
1139    pub signal: Option<i32>,
1140    pub at_ms: u64,
1141}
1142
1143/// Point-in-time module status answerable by subc without forwarding to the
1144/// module process.
1145#[derive(Debug, Clone, PartialEq)]
1146pub struct ModuleStatus {
1147    pub module_id: String,
1148    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1149    pub waiting_for_previous_pid: Option<u32>,
1150    /// Why the previous instance still prevents a new start.
1151    pub waiting_for_previous_detail: Option<String>,
1152    /// The daemon's Windows per-spawn record: `pipe` after the process it
1153    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1154    /// List/status reports use this record, not the source the module sends
1155    /// in its registration message's manifest.provenance.launch_nonce_source.
1156    pub launch_nonce_source: Option<String>,
1157    pub state: ModuleState,
1158    pub enabled: bool,
1159    pub process_alive: bool,
1160    pub registration_active: bool,
1161    /// The module's declared wire protocol, carried beside `live` because it is
1162    /// what makes `live` readable: the two fields answer one question together.
1163    /// While a process is alive this is its launch declaration, not a later
1164    /// pending-reload edit. When down it is the configured next launch protocol.
1165    pub protocol: ModuleProtocol,
1166    /// Whether the module is serving, under the strongest definition the daemon
1167    /// can assert for its protocol.
1168    ///
1169    /// A subc module must also be REGISTERED: its process being alive says
1170    /// nothing about whether it can take a request. A `protocol: "none"` module
1171    /// never registers, so that term is dropped and this falls back to "enabled,
1172    /// running, and the process the daemon launched is alive" -- which is all
1173    /// the daemon observes about a process that speaks no subc wire. It stays a
1174    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1175    /// rather than printing it bare.
1176    pub live: bool,
1177    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1178    /// restarts have already released their slot, so this count can go down
1179    /// without anybody touching the module.
1180    pub restart_count: u32,
1181    /// Replacement processes spawned over this module's entire supervisor lifetime;
1182    /// unlike `restart_count`, this value is never reset by an operator action
1183    /// and never falls out of a window.
1184    pub lifetime_restarts: u32,
1185    pub spawn_generation: u64,
1186    /// The budget `restart_count` is spent against. Carried alongside the count
1187    /// because the count alone does not say how close the module is to being
1188    /// disabled, and reporting one without the other is what makes an
1189    /// about-to-be-retired module look ordinary.
1190    pub max_restarts: u32,
1191    /// The span `restart_count` is counted over. Carried with the pair above for
1192    /// the same reason they are carried together: "2 of 3" means one thing for a
1193    /// ten-minute window and something else entirely for a lifetime.
1194    pub restart_window: Duration,
1195    /// Effective drain and restart timing policy used by this running module.
1196    /// These values are carried together with the restart budget so status
1197    /// readers can compare configured intent with what the supervisor applied.
1198    pub drain_timeout: Duration,
1199    pub restart_backoff: Duration,
1200    pub restart_max_backoff: Duration,
1201    /// The module's process. On macOS this stays absent while the `ck-subc`
1202    /// launch trampoline is still running in that pid, and appears once the
1203    /// exec acknowledgement confirms the module image has replaced it. Launch
1204    /// time and the supervisor's own hold on the process are unaffected.
1205    pub pid: Option<u32>,
1206    pub spawned_at_ms: Option<u64>,
1207    pub spawned_from: Option<PathBuf>,
1208    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1209    pub process_start_time: Option<u64>,
1210    pub last_exit: Option<ExitReport>,
1211    pub health: ModuleHealthStatus,
1212}
1213
1214#[cfg(test)]
1215#[derive(Debug, Clone, Copy, PartialEq)]
1216struct ActorSelectCheckpoint {
1217    generation: u64,
1218    turn: u64,
1219    registered_connection: Option<ConnectionId>,
1220    next_probe_at: Option<Instant>,
1221    wake_after: Option<Duration>,
1222}
1223
1224#[derive(Debug, Clone, PartialEq)]
1225struct SupervisorSnapshot {
1226    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1227    #[cfg(windows)]
1228    pipe_consumed: Option<Arc<OnceLock<()>>>,
1229    /// Counts running-child loop turns, not executor polls of a parked wait.
1230    #[cfg(test)]
1231    actor_turns: u64,
1232    /// Running select after exec confirmation and with no unconsumed registry
1233    /// event. Recording a spawn or starting a loop turn is not this barrier.
1234    #[cfg(test)]
1235    actor_select: Option<ActorSelectCheckpoint>,
1236    state: ModuleState,
1237    enabled: bool,
1238    process_alive: bool,
1239    spawned_protocol: Option<ModuleProtocol>,
1240    spawn_failure: Option<String>,
1241    waiting_for_previous_pid: Option<u32>,
1242    waiting_for_previous_detail: Option<String>,
1243    /// When each crash restart was spent, oldest first. This IS the crash
1244    /// budget: its in-window length is the count an operator sees and the count
1245    /// the restart decision is made against, so there is no second counter that
1246    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1247    /// operator actions that used to zero the old lifetime counter.
1248    crash_restarts: VecDeque<Instant>,
1249    lifetime_restarts: u32,
1250    /// Successful child spawns in this daemon incarnation.
1251    ///
1252    /// `lifetime_restarts` was considered and rejected: it starts at zero
1253    /// (line 640), successful initial/operator spawns in `set_running` do not
1254    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1255    /// increments before a successful replacement exists (lines 604, 3846,
1256    /// and 3921), so a failed spawn can consume it. This counter moves only
1257    /// when a live PID is accepted below.
1258    spawn_generation: u64,
1259    pid: Option<u32>,
1260    #[cfg(target_os = "macos")]
1261    report_ready: Option<Arc<OnceLock<()>>>,
1262    /// Last reaped child, retained after current process facts are cleared.
1263    reaped_pid: Option<u32>,
1264    /// Whether a replacement has already been scheduled for the current process.
1265    respawn_pending: bool,
1266    /// A second restart is waiting for the replacement already scheduled.
1267    coalesced_restart_pending: bool,
1268    spawned_at_ms: Option<u64>,
1269    spawned_from: Option<PathBuf>,
1270    launch_spec_digest: Option<LaunchSpecDigest>,
1271    spawned_file_identity: Option<SpawnedFileIdentity>,
1272    process_start_time: Option<u64>,
1273    deliberate_severance: Option<ProcessIdentity>,
1274    last_exit: Option<ExitReport>,
1275    /// Diagnostic attached to the next drain's terminal record, if any.
1276    drain_disposition_detail: Option<String>,
1277    health: ModuleHealthStatus,
1278    /// Whether the current process was started as a swap candidate and so
1279    /// lives in the module's alternate cgroup. The next swap's candidate takes
1280    /// the other one, so the two processes of a swap never share a cgroup. A
1281    /// plain spawn always uses the primary cgroup.
1282    in_alternate_slot: bool,
1283    /// Whether the current `Draining` state ends in a replacement process
1284    /// (restart, reload, health restart) rather than a stop. Only meaningful
1285    /// while `state` is `Draining`; every entry into that state rewrites it.
1286    /// It is what lets route.open answer the retryable `module_reloading` to a
1287    /// consumer that reaches a still-registered process mid-restart, instead of
1288    /// the `supervisor_not_live` a stop or disable deserves.
1289    draining_to_replace: bool,
1290    /// Whether a configuration update has been applied since the current
1291    /// process was spawned, so that process runs an older spec than the one
1292    /// the supervisor now holds. A queued restart is only coalesced into a
1293    /// fresher process when this is false: a restart requested to pick up a
1294    /// new configuration must not be satisfied by a process that predates it.
1295    configuration_updated_since_spawn: bool,
1296}
1297
1298impl SupervisorSnapshot {
1299    /// The pid that status, provenance and resource readings may report. While
1300    /// the launch trampoline still runs in the pid, reading its executable or
1301    /// resource use would describe `ck-subc`, not the module, so none is
1302    /// reported until the exec acknowledgement confirms the module image.
1303    fn reported_pid(&self) -> Option<u32> {
1304        #[cfg(target_os = "macos")]
1305        if self
1306            .report_ready
1307            .as_ref()
1308            .is_some_and(|ready| ready.get().is_none())
1309        {
1310            return None;
1311        }
1312        self.pid
1313    }
1314
1315    fn starting() -> Self {
1316        Self::new(ModuleState::Starting, true)
1317    }
1318
1319    fn disabled() -> Self {
1320        Self::new(ModuleState::Disabled, false)
1321    }
1322
1323    /// Crash restarts still inside `window`, having dropped the ones that are
1324    /// not. Pruning on read is what makes the budget a rate: an instant older
1325    /// than the window stops holding a slot the moment anybody counts.
1326    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1327        while let Some(oldest) = self.crash_restarts.front() {
1328            if now.duration_since(*oldest) > window {
1329                self.crash_restarts.pop_front();
1330            } else {
1331                break;
1332            }
1333        }
1334        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1335    }
1336
1337    /// Spend one unit of the crash budget and record the restart in the ledger.
1338    ///
1339    /// The ring is bounded by the cap because more than `max_restarts` in-window
1340    /// instants can never be reached (the caller refuses the restart first), so
1341    /// anything beyond that is an unbounded queue waiting to happen.
1342    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1343        self.crash_restarts.push_back(now);
1344        while self.crash_restarts.len() > policy.max_restarts as usize {
1345            self.crash_restarts.pop_front();
1346        }
1347        self.lifetime_restarts += 1;
1348    }
1349
1350    /// Reserve one crash-restart slot and calculate the delay before respawning.
1351    /// The count is captured before recording this restart, so the first retry
1352    /// uses the base delay and each later in-window retry escalates once.
1353    fn next_crash_restart(
1354        &mut self,
1355        policy: &RestartPolicy,
1356        now: Instant,
1357    ) -> Option<CrashRestartSchedule> {
1358        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1359        if restart_in_window >= policy.max_restarts {
1360            return None;
1361        }
1362        self.record_crash_restart(policy, now);
1363        Some(CrashRestartSchedule {
1364            restart_in_window,
1365            delay: policy.delay_for_restart(restart_in_window),
1366        })
1367    }
1368
1369    /// Give the module its full budget back, as an operator restart, reload, or
1370    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1371    /// ledger of what actually happened, and an operator action does not unmake
1372    /// the crashes.
1373    fn clear_crash_restarts(&mut self) {
1374        self.crash_restarts.clear();
1375    }
1376
1377    fn new(state: ModuleState, enabled: bool) -> Self {
1378        Self {
1379            exec_outcome: None,
1380            #[cfg(test)]
1381            actor_turns: 0,
1382            #[cfg(test)]
1383            actor_select: None,
1384            state,
1385            enabled,
1386            process_alive: false,
1387            #[cfg(windows)]
1388            pipe_consumed: None,
1389            spawned_protocol: None,
1390            spawn_failure: None,
1391            waiting_for_previous_pid: None,
1392            waiting_for_previous_detail: None,
1393            crash_restarts: VecDeque::new(),
1394            lifetime_restarts: 0,
1395            spawn_generation: 0,
1396            pid: None,
1397            #[cfg(target_os = "macos")]
1398            report_ready: None,
1399            reaped_pid: None,
1400            respawn_pending: false,
1401            coalesced_restart_pending: false,
1402            spawned_at_ms: None,
1403            spawned_from: None,
1404            launch_spec_digest: None,
1405            spawned_file_identity: None,
1406            process_start_time: None,
1407            deliberate_severance: None,
1408            last_exit: None,
1409            drain_disposition_detail: None,
1410            health: ModuleHealthStatus::default(),
1411            in_alternate_slot: false,
1412            draining_to_replace: false,
1413            configuration_updated_since_spawn: false,
1414        }
1415    }
1416}
1417
1418type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1419
1420type SpawnSubscriberKey = (ConnectionId, u64);
1421
1422#[derive(Debug)]
1423struct SpawnSubscriber {
1424    version: u8,
1425    frames: mpsc::Sender<Frame>,
1426    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1427    /// from which event. The full frame channel cannot carry that news, so it
1428    /// travels beside it; see `SpawnEventFeed::subscribe`.
1429    lagged: Option<oneshot::Sender<SpawnCursor>>,
1430}
1431
1432#[derive(Debug)]
1433struct SpawnEventState {
1434    daemon_incarnation: String,
1435    seq: u64,
1436    capacity: usize,
1437    live: HashMap<String, LiveSpawn>,
1438    generations: HashMap<String, u64>,
1439    events: VecDeque<SpawnEvent>,
1440    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1441}
1442
1443impl Default for SpawnEventState {
1444    fn default() -> Self {
1445        Self {
1446            daemon_incarnation: "unconfigured".to_string(),
1447            seq: 0,
1448            capacity: SPAWN_EVENT_RING_CAPACITY,
1449            live: HashMap::new(),
1450            generations: HashMap::new(),
1451            events: VecDeque::new(),
1452            subscribers: HashMap::new(),
1453        }
1454    }
1455}
1456
1457#[derive(Debug, Clone, Default)]
1458struct SpawnEventFeed(
1459    Arc<Mutex<SpawnEventState>>,
1460    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1461);
1462
1463#[derive(Debug, Clone, PartialEq, Eq)]
1464pub(crate) enum SpawnSubscribeRefusal {
1465    ForeignIncarnation { current: String },
1466    TooOld { oldest: SpawnCursor },
1467    Frame(String),
1468}
1469
1470impl SpawnEventFeed {
1471    fn configure_incarnation(&self, daemon_incarnation: String) {
1472        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1473        state.daemon_incarnation = daemon_incarnation;
1474        state.seq = 0;
1475        state.live.clear();
1476        state.generations.clear();
1477        state.events.clear();
1478        state.subscribers.clear();
1479    }
1480
1481    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1482        SpawnCursor {
1483            daemon_incarnation: state.daemon_incarnation.clone(),
1484            seq: state.seq,
1485        }
1486    }
1487
1488    fn snapshot(&self) -> SpawnSnapshot {
1489        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1490        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1491        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1492        SpawnSnapshot {
1493            cursor: Self::cursor(&state),
1494            ring_bound: state.capacity as u64,
1495            live,
1496        }
1497    }
1498
1499    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1500        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1501        let generation = state
1502            .generations
1503            .get(module_id)
1504            .copied()
1505            .unwrap_or(0)
1506            .checked_add(1)
1507            .expect("spawn generation exhausted");
1508        state.generations.insert(module_id.to_string(), generation);
1509        let live = LiveSpawn {
1510            module_id: module_id.to_string(),
1511            spawn_generation: generation,
1512            pid,
1513            spawned_at_ms,
1514        };
1515        state.live.insert(module_id.to_string(), live);
1516        Self::emit_locked(
1517            &mut state,
1518            SpawnEventKind::Spawned,
1519            module_id.to_string(),
1520            generation,
1521            pid,
1522            None,
1523            None,
1524        );
1525        generation
1526    }
1527
1528    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1529        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1530        let Some(live) = state.live.remove(module_id) else {
1531            warn!(
1532                module_id,
1533                "terminal record had no live spawn event identity"
1534            );
1535            return;
1536        };
1537        Self::emit_locked(
1538            &mut state,
1539            SpawnEventKind::Exited,
1540            module_id.to_string(),
1541            live.spawn_generation,
1542            live.pid,
1543            exit_code,
1544            exit_signal,
1545        );
1546    }
1547
1548    /// Report the exit of a process that a swap has already replaced.
1549    ///
1550    /// `emit_exited` removes the module's live entry, which after a swap's
1551    /// cutover describes the promoted candidate, not the old process now
1552    /// exiting. This emits the old generation's exit and leaves the live entry
1553    /// alone unless it still names that generation.
1554    fn emit_superseded_exited(
1555        &self,
1556        module_id: &str,
1557        spawn_generation: u64,
1558        pid: u32,
1559        exit_code: Option<i32>,
1560        exit_signal: Option<i32>,
1561    ) {
1562        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1563        if state
1564            .live
1565            .get(module_id)
1566            .is_some_and(|live| live.spawn_generation == spawn_generation)
1567        {
1568            state.live.remove(module_id);
1569        }
1570        Self::emit_locked(
1571            &mut state,
1572            SpawnEventKind::Exited,
1573            module_id.to_string(),
1574            spawn_generation,
1575            pid,
1576            exit_code,
1577            exit_signal,
1578        );
1579    }
1580
1581    #[allow(clippy::too_many_arguments)]
1582    fn emit_locked(
1583        state: &mut SpawnEventState,
1584        kind: SpawnEventKind,
1585        module_id: String,
1586        spawn_generation: u64,
1587        pid: u32,
1588        exit_code: Option<i32>,
1589        exit_signal: Option<i32>,
1590    ) {
1591        state.seq = state
1592            .seq
1593            .checked_add(1)
1594            .expect("spawn event sequence exhausted");
1595        let event = SpawnEvent {
1596            cursor: Self::cursor(state),
1597            kind,
1598            module_id,
1599            spawn_generation,
1600            pid,
1601            exit_code,
1602            exit_signal,
1603        };
1604        state.events.push_back(event.clone());
1605        while state.events.len() > state.capacity {
1606            state.events.pop_front();
1607        }
1608        let body = match serde_json::to_vec(&event) {
1609            Ok(body) => body,
1610            Err(error) => {
1611                error!(%error, "failed to serialize supervisor spawn event");
1612                return;
1613            }
1614        };
1615        state.subscribers.retain(|(connection_id, corr), subscriber| {
1616            let frame = Frame::build_with_version(
1617                subscriber.version,
1618                FrameType::StreamData,
1619                control_flags(),
1620                0,
1621                0,
1622                *corr,
1623                body.clone(),
1624            );
1625            match frame {
1626                Ok(frame) => {
1627                    if subscriber.frames.try_send(frame).is_ok() {
1628                        true
1629                    } else {
1630                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1631                        if let Some(lagged) = subscriber.lagged.take() {
1632                            let _ = lagged.send(event.cursor.clone());
1633                        }
1634                        false
1635                    }
1636                }
1637                Err(error) => {
1638                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1639                    false
1640                }
1641            }
1642        });
1643    }
1644
1645    fn subscribe(
1646        &self,
1647        connection_id: ConnectionId,
1648        corr: u64,
1649        version: u8,
1650        since: Option<SpawnCursor>,
1651        sink: FrameSink,
1652    ) -> Result<(), SpawnSubscribeRefusal> {
1653        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1654        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1655        {
1656            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1657            let replay = if let Some(since) = since {
1658                if since.daemon_incarnation != state.daemon_incarnation {
1659                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1660                        current: state.daemon_incarnation.clone(),
1661                    });
1662                }
1663                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1664                    if since.seq < oldest.seq.saturating_sub(1) {
1665                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1666                    }
1667                }
1668                state
1669                    .events
1670                    .iter()
1671                    .filter(|event| event.cursor.seq > since.seq)
1672                    .cloned()
1673                    .collect::<Vec<_>>()
1674            } else {
1675                Vec::new()
1676            };
1677            for event in replay {
1678                let body = serde_json::to_vec(&event)
1679                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1680                let frame = Frame::build_with_version(
1681                    version,
1682                    FrameType::StreamData,
1683                    control_flags(),
1684                    0,
1685                    0,
1686                    corr,
1687                    body,
1688                )
1689                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1690                frames
1691                    .try_send(frame)
1692                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1693            }
1694            state.subscribers.insert(
1695                (connection_id, corr),
1696                SpawnSubscriber {
1697                    version,
1698                    frames,
1699                    lagged: Some(lagged),
1700                },
1701            );
1702        }
1703        // The lagged terminal is sent here, by the forwarder, rather than by
1704        // the emitter: at the moment of the drop the subscriber's own channel
1705        // is full, and writing to the connection sink directly from the emitter
1706        // would put the Error AHEAD of the events still queued in that channel
1707        // (and the emitter holds the feed lock, so it cannot await the sink).
1708        // Dropping the subscriber drops the only sender, so `recv` drains every
1709        // queued event and then returns `None`; only then is the Error sent, so
1710        // the client sees each event it can keep, then the reason it was cut.
1711        // Cancel and connection removal drop the oneshot unsent, so they end
1712        // the stream with no Error.
1713        tokio::spawn(async move {
1714            while let Some(frame) = receiver.recv().await {
1715                if sink.send(frame).await.is_err() {
1716                    return;
1717                }
1718            }
1719            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1720                return;
1721            };
1722            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1723                Ok(frame) => {
1724                    let _ = sink.send(frame).await;
1725                }
1726                Err(error) => {
1727                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1728                }
1729            }
1730        });
1731        Ok(())
1732    }
1733
1734    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1735        let Some(subscriber) = self
1736            .0
1737            .lock()
1738            .unwrap_or_else(|p| p.into_inner())
1739            .subscribers
1740            .remove(&(connection_id, corr))
1741        else {
1742            return false;
1743        };
1744        if let Ok(frame) = Frame::build_with_version(
1745            subscriber.version,
1746            FrameType::StreamEnd,
1747            control_flags(),
1748            0,
1749            0,
1750            corr,
1751            Vec::new(),
1752        ) {
1753            tokio::spawn(async move {
1754                let _ = subscriber.frames.send(frame).await;
1755            });
1756        }
1757        true
1758    }
1759
1760    fn remove_connection(&self, connection_id: ConnectionId) {
1761        self.0
1762            .lock()
1763            .unwrap_or_else(|p| p.into_inner())
1764            .subscribers
1765            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1766    }
1767
1768    #[cfg(any(test, feature = "test-support"))]
1769    fn set_capacity(&self, capacity: usize) {
1770        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1771    }
1772
1773    #[cfg(any(test, feature = "test-support"))]
1774    fn subscriber_count(&self) -> usize {
1775        self.0
1776            .lock()
1777            .unwrap_or_else(|p| p.into_inner())
1778            .subscribers
1779            .len()
1780    }
1781}
1782
1783/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1784/// The terminal Error a lagged spawn subscriber receives after its queued events.
1785fn spawn_subscriber_lagged_frame(
1786    version: u8,
1787    corr: u64,
1788    first_undelivered: SpawnCursor,
1789) -> Result<Frame, String> {
1790    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1791        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1792        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1793            .to_string(),
1794        detail: Some(serde_json::json!({
1795            "first_undelivered_cursor": first_undelivered
1796        })),
1797    })
1798    .map_err(|error| error.to_string())?;
1799    Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1800        .map_err(|error| error.to_string())
1801}
1802
1803pub trait ModuleProcessLiveness: Send + Sync {
1804    fn process_live(&self, module_id: &str) -> Option<bool>;
1805
1806    /// Whether the supervisor is replacing this module's process right now: an
1807    /// operator restart or reload, a health restart, or a crash respawn whose
1808    /// backoff is running. A module in that state is not live, but a consumer
1809    /// refused now should retry shortly rather than treat the target as gone.
1810    /// Stopped, failed, and disabled modules are not replacing.
1811    fn process_replacing(&self, _module_id: &str) -> bool {
1812        false
1813    }
1814}
1815
1816/// Shared process-liveness registry keyed by supervised `module_id`.
1817#[derive(Debug, Clone, Default)]
1818pub struct SupervisorProcessLiveness {
1819    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1820}
1821
1822impl SupervisorProcessLiveness {
1823    pub fn new() -> Self {
1824        Self::default()
1825    }
1826
1827    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1828        let mut snapshots = self
1829            .snapshots
1830            .lock()
1831            .unwrap_or_else(|poisoned| poisoned.into_inner());
1832        snapshots.insert(module_id, snapshot);
1833    }
1834
1835    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1836        let mut snapshots = self
1837            .snapshots
1838            .lock()
1839            .unwrap_or_else(|poisoned| poisoned.into_inner());
1840        let is_current = snapshots
1841            .get(module_id)
1842            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1843            .unwrap_or(false);
1844        if is_current {
1845            snapshots.remove(module_id);
1846        }
1847    }
1848}
1849
1850impl ModuleProcessLiveness for SupervisorProcessLiveness {
1851    fn process_live(&self, module_id: &str) -> Option<bool> {
1852        let snapshot = {
1853            let snapshots = self
1854                .snapshots
1855                .lock()
1856                .unwrap_or_else(|poisoned| poisoned.into_inner());
1857            snapshots.get(module_id).cloned()
1858        }?;
1859        let snapshot = snapshot
1860            .lock()
1861            .unwrap_or_else(|poisoned| poisoned.into_inner());
1862        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1863    }
1864
1865    fn process_replacing(&self, module_id: &str) -> bool {
1866        let Some(snapshot) = self
1867            .snapshots
1868            .lock()
1869            .unwrap_or_else(|poisoned| poisoned.into_inner())
1870            .get(module_id)
1871            .cloned()
1872        else {
1873            return false;
1874        };
1875        let snapshot = snapshot
1876            .lock()
1877            .unwrap_or_else(|poisoned| poisoned.into_inner());
1878        snapshot.enabled
1879            && match snapshot.state {
1880                ModuleState::Restarting => true,
1881                ModuleState::Draining => snapshot.draining_to_replace,
1882                ModuleState::Starting
1883                | ModuleState::Running
1884                | ModuleState::Unresponsive
1885                | ModuleState::Stopped
1886                | ModuleState::Failed
1887                | ModuleState::Disabled => false,
1888            }
1889    }
1890}
1891
1892#[cfg(test)]
1893#[derive(Debug, Default)]
1894struct ReloadExitRecordGate {
1895    reached: tokio::sync::Notify,
1896    resume: tokio::sync::Notify,
1897}
1898
1899#[derive(Debug, Clone, Copy)]
1900enum RespawnKind {
1901    Spawn,
1902    Reload,
1903}
1904
1905#[derive(Debug, Clone, Copy)]
1906struct PendingRespawn {
1907    deadline: Instant,
1908    kind: RespawnKind,
1909}
1910
1911type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1912
1913#[derive(Debug, Clone)]
1914struct SupervisorRuntimeConfig {
1915    start_requester: Option<Principal>,
1916    pending_activity_restart: bool,
1917    provenance_probe: ExecutableIdentityProbe,
1918    /// Restart operations hand their backoff to the loop, which keeps serving commands.
1919    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1920    /// A reload acknowledges completion only after its replacement registers.
1921    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1922    restart_policy: RestartPolicy,
1923    /// This module's RESOLVED drain budget: per-module config when present,
1924    /// else `default_drain_timeout`.
1925    drain_timeout: Duration,
1926    /// Shared with the status handle so the attested value changes atomically
1927    /// when a rescan updates the running drain policy.
1928    effective_drain_timeout: Arc<Mutex<Duration>>,
1929    /// The supervisor-wide fallback, kept so a configuration update that
1930    /// REMOVES the per-module override can re-resolve to it.
1931    default_drain_timeout: Duration,
1932    health: HealthConfig,
1933    connection_file_path: Option<PathBuf>,
1934    capture_logs_dir: Option<PathBuf>,
1935    forwarding: Option<Arc<ForwardingTable>>,
1936    /// The shared handle, so every spawn path (initial, restart, reload) records the
1937    /// reserved-module launch nonce the HELLO verifier checks against.
1938    supervisor_handle: Option<SupervisorHandle>,
1939    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
1940    /// status queries.
1941    ///
1942    /// One ring per module, held across every respawn. The lines explaining an exit
1943    /// are written BEFORE that exit, so a ring recreated per process would be empty
1944    /// exactly when it is asked for.
1945    stderr_ring: Arc<Mutex<StderrRing>>,
1946    terminal_ring: Arc<Mutex<TerminalRing>>,
1947    spawn_events: SpawnEventFeed,
1948    child_roster: ChildRoster,
1949    #[cfg(target_os = "linux")]
1950    cgroup_placement: Option<subc_cgroup::Placement>,
1951    #[cfg(test)]
1952    test_seed_stale_facts_before_enable_spawn: bool,
1953    #[cfg(test)]
1954    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1955}
1956
1957#[derive(Debug, Clone, PartialEq, Eq)]
1958struct SupervisedConfiguration {
1959    spec: ModuleSpec,
1960    health: HealthConfig,
1961}
1962
1963/// Shared daemon lookup table for supervised module handles.
1964///
1965/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
1966/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
1967/// launch nonces recorded at spawn are checked by the same daemon instance.
1968#[derive(Debug, Clone, Default)]
1969pub struct SupervisorHandle {
1970    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1971    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1972    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1973    /// Module ids the supervisor has taken on. An id is added BEFORE the
1974    /// module's first process is spawned and removed only when the module
1975    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
1976    /// the keys of `modules`.
1977    ///
1978    /// `modules` cannot answer "is this module configured?" on its own: a
1979    /// [`SupervisedModule`] only exists once its process has been spawned, and
1980    /// a fast child can connect, register, sync its scopes and ask about them
1981    /// before the supervisor has inserted it. Answering "not configured" in that
1982    /// gap makes scope admission refuse with the terminal "will never sync"
1983    /// instead of the retryable "has not synced yet".
1984    configured_ids: Arc<Mutex<HashSet<String>>>,
1985    spawn_events: SpawnEventFeed,
1986    /// The current expected launch nonce for each reserved module_id. Set when the
1987    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
1988    /// non-reserved module never has an entry here and is never nonce-checked.
1989    /// Reserved module ids and the nonce that authorizes their next HELLO.
1990    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
1991    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
1992    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
1993    /// had NO entry and admitted anyone: the reservation protected the nonce
1994    /// holder, not the NAME (found live by CKCRED's canary probe registering
1995    /// against a reserved scratch id).
1996    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
1997    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
1998    ///
1999    /// This is deliberately in-memory only: subc is state-free across daemon
2000    /// restarts, and the tombstone only explains the hours-after-removal window
2001    /// while this executing daemon is still alive. Do not persist it in a store.
2002    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2003    /// The current launch nonce for every supervised spawn. This is separate from
2004    /// reserved_nonces because consumer route.open attestation applies to all spawned
2005    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2006    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2007    /// Reserved namespace prefixes mapped to the supervised owner module whose
2008    /// current spawn nonce authorizes HELLO claims below the prefix.
2009    ///
2010    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2011    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2012    /// accidental collisions and lower-trust processes from squatting protected
2013    /// namespaces.
2014    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2015    /// Blue/green swaps in progress, by module id. An entry exists from just
2016    /// before the candidate process is spawned until the swap has failed, or
2017    /// has cut over and the old process is gone. While it exists, HELLO for the
2018    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2019    /// consumer attestation accepts both processes' nonces.
2020    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2021    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2022    promotion_observer: PromotionObserverSlot,
2023    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2024    /// this daemon-wide ordering, a rescan could retire or update a module while a
2025    /// concurrent reload still held its old handle and launch specification.
2026    operation_lock: Arc<AsyncMutex<()>>,
2027}
2028
2029/// Told when a swap has promoted its candidate to be the module's active
2030/// registration.
2031///
2032/// An ordinary HELLO runs the control plane's registration side effects (the
2033/// capability cache, the deny census, the requirement recompute) as it
2034/// registers. A swap candidate's HELLO does not, because it is not routable;
2035/// promotion is when those must run instead, and promotion happens in the
2036/// supervisor, which has no other way into the control handler.
2037pub(crate) trait SwapPromotionObserver: Send + Sync {
2038    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2039}
2040
2041/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2042/// control handler) owns this handle, so a strong reference back would be a
2043/// cycle that keeps both alive.
2044#[derive(Clone, Default)]
2045struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2046
2047impl fmt::Debug for PromotionObserverSlot {
2048    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2049        f.write_str("PromotionObserverSlot")
2050    }
2051}
2052
2053/// The nonces of one open swap.
2054#[derive(Debug, Clone)]
2055struct OpenSwap {
2056    /// The launch nonce minted for the candidate process. It is the swap
2057    /// token: the only thing that admits a HELLO into the candidate slot.
2058    candidate_nonce: String,
2059    /// The incumbent's launch nonce, captured when the swap opened. It is kept
2060    /// here because cutover moves the module's recorded spawn nonce to the
2061    /// candidate while the incumbent is still draining and its consumers are
2062    /// still attesting with this one.
2063    incumbent_nonce: Option<String>,
2064    /// Set once a HELLO has been admitted with the swap token, so the token
2065    /// admits one registration and cannot be replayed after cutover empties
2066    /// the candidate slot.
2067    candidate_admitted: bool,
2068}
2069
2070/// What the swap gate says about a HELLO. See
2071/// [`SupervisorHandle::swap_hello_admission`].
2072#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2073pub(crate) enum SwapHelloAdmission {
2074    /// No swap is open for the id (or the HELLO carries the incumbent's own
2075    /// nonce); the ordinary gates decide.
2076    NotSwapping,
2077    /// The HELLO carries the swap token: register it into the candidate slot.
2078    Candidate,
2079    /// A swap is open and the HELLO carries a nonce the supervisor did not
2080    /// mint for this id, no nonce, or a token already used.
2081    Refused,
2082}
2083
2084#[derive(Debug, Clone, PartialEq, Eq)]
2085pub(crate) enum ReservedHelloRejection {
2086    Exact {
2087        module_id: String,
2088    },
2089    Prefix {
2090        prefix: String,
2091        owner_module_id: String,
2092    },
2093}
2094
2095impl SupervisorHandle {
2096    pub fn new() -> Self {
2097        Self::default()
2098    }
2099
2100    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2101        self.spawn_events.snapshot()
2102    }
2103
2104    pub(crate) fn subscribe_spawns(
2105        &self,
2106        connection_id: ConnectionId,
2107        corr: u64,
2108        version: u8,
2109        since: Option<SpawnCursor>,
2110        sink: FrameSink,
2111    ) -> Result<(), SpawnSubscribeRefusal> {
2112        self.spawn_events
2113            .subscribe(connection_id, corr, version, since, sink)
2114    }
2115
2116    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2117        self.spawn_events.cancel(connection_id, corr)
2118    }
2119
2120    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2121        self.spawn_events.remove_connection(connection_id);
2122    }
2123
2124    #[cfg(any(test, feature = "test-support"))]
2125    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2126        assert!(capacity > 0, "spawn event capacity must be non-zero");
2127        self.spawn_events.set_capacity(capacity);
2128    }
2129
2130    #[cfg(any(test, feature = "test-support"))]
2131    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2132        self.spawn_events.subscriber_count()
2133    }
2134
2135    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2136    /// a respawn invalidates stale consumer identities.
2137    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2138        self.spawn_nonces
2139            .lock()
2140            .unwrap_or_else(|poisoned| poisoned.into_inner())
2141            .insert(module_id.to_string(), nonce);
2142    }
2143
2144    /// Record the launch nonce expected from the next HELLO for a reserved module,
2145    /// replacing any prior nonce (a respawn invalidates the previous one).
2146    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2147        self.reserved_nonces
2148            .lock()
2149            .unwrap_or_else(|poisoned| poisoned.into_inner())
2150            .insert(module_id.to_string(), Some(nonce));
2151    }
2152
2153    /// Record namespace prefixes owned by a supervised module.
2154    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2155        let mut owners = self
2156            .reserved_prefix_owners
2157            .lock()
2158            .unwrap_or_else(|poisoned| poisoned.into_inner());
2159        owners.retain(|_, owner| owner != owner_module_id);
2160        for prefix in prefixes {
2161            owners.insert(prefix.clone(), owner_module_id.to_string());
2162        }
2163    }
2164
2165    /// The launch nonce most recently minted for a module's spawn, if any.
2166    #[cfg(test)]
2167    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2168        self.spawn_nonces
2169            .lock()
2170            .unwrap_or_else(|poisoned| poisoned.into_inner())
2171            .get(module_id)
2172            .cloned()
2173    }
2174
2175    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2176        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2177        let spawn_nonce = self
2178            .spawn_nonces
2179            .lock()
2180            .unwrap_or_else(|poisoned| poisoned.into_inner())
2181            .get(&spec.module_id)
2182            .cloned();
2183        let mut reserved_nonces = self
2184            .reserved_nonces
2185            .lock()
2186            .unwrap_or_else(|poisoned| poisoned.into_inner());
2187        if spec.reserved {
2188            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2189            // reserved name whose module has never spawned has no legitimate
2190            // holder, and the entry's absence is what used to leave the name
2191            // open to the first claimant.
2192            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2193        }
2194        drop(reserved_nonces);
2195        // A later unreserved declaration must not silently unreserve an id that
2196        // was retained after its reserved configuration was removed. The explicit
2197        // release ceremony is the only operation that retires that gate.
2198        self.removal_tombstones
2199            .lock()
2200            .unwrap_or_else(|poisoned| poisoned.into_inner())
2201            .remove(&spec.module_id);
2202    }
2203
2204    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2205    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2206    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2207    /// with no matching prefix are always authorized.
2208    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2209        self.reserved_hello_rejection(module_id, presented)
2210            .is_none()
2211    }
2212
2213    pub(crate) fn reserved_hello_rejection(
2214        &self,
2215        module_id: &str,
2216        presented: Option<&str>,
2217    ) -> Option<ReservedHelloRejection> {
2218        let nonces = self
2219            .reserved_nonces
2220            .lock()
2221            .unwrap_or_else(|poisoned| poisoned.into_inner());
2222        if let Some(expected) = nonces.get(module_id) {
2223            // `None` = reserved with no legitimate holder: refuse every
2224            // presentation, because no process can hold a nonce that was never
2225            // minted. Only a real minted nonce admits, in constant time.
2226            let authorized = match expected {
2227                Some(expected) => {
2228                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2229                }
2230                None => false,
2231            };
2232            if authorized {
2233                return None;
2234            }
2235            return Some(ReservedHelloRejection::Exact {
2236                module_id: module_id.to_string(),
2237            });
2238        }
2239        drop(nonces);
2240
2241        let matched_prefix = self
2242            .reserved_prefix_owners
2243            .lock()
2244            .unwrap_or_else(|poisoned| poisoned.into_inner())
2245            .iter()
2246            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2247            .max_by_key(|(prefix, _)| prefix.len())
2248            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2249        let (prefix, owner_module_id) = matched_prefix?;
2250
2251        let authorized = presented.is_some_and(|presented| {
2252            self.spawn_nonces
2253                .lock()
2254                .unwrap_or_else(|poisoned| poisoned.into_inner())
2255                .get(&owner_module_id)
2256                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2257                // While the owner is being swapped, children started by
2258                // either of its two processes hold that process's nonce.
2259                || self.swap_nonce_matches(&owner_module_id, presented)
2260        });
2261        if authorized {
2262            None
2263        } else {
2264            Some(ReservedHelloRejection::Prefix {
2265                prefix,
2266                owner_module_id,
2267            })
2268        }
2269    }
2270
2271    /// Whether a consumer connection proved it came from a daemon-spawned module.
2272    ///
2273    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2274    /// accepted only for module ids the supervisor has spawned.
2275    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2276        if presented.is_empty() {
2277            return false;
2278        }
2279        let nonces = self
2280            .spawn_nonces
2281            .lock()
2282            .unwrap_or_else(|poisoned| poisoned.into_inner());
2283        let current = nonces
2284            .get(module_id)
2285            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2286        drop(nonces);
2287        // During a swap two processes of the module are alive, and a consumer
2288        // started by either one presents that process's nonce. Accepting only
2289        // the recorded one would fail the incumbent's consumers for the whole
2290        // overlap once cutover moves the record to the candidate.
2291        current || self.swap_nonce_matches(module_id, presented)
2292    }
2293
2294    /// Whether `presented` is either nonce of an open swap for `module_id`.
2295    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2296        let swaps = self
2297            .swaps
2298            .lock()
2299            .unwrap_or_else(|poisoned| poisoned.into_inner());
2300        swaps.get(module_id).is_some_and(|swap| {
2301            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2302                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2303                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2304                })
2305        })
2306    }
2307
2308    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2309    /// Called before the candidate process exists.
2310    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2311        let incumbent_nonce = self
2312            .spawn_nonces
2313            .lock()
2314            .unwrap_or_else(|poisoned| poisoned.into_inner())
2315            .get(module_id)
2316            .cloned();
2317        self.swaps
2318            .lock()
2319            .unwrap_or_else(|poisoned| poisoned.into_inner())
2320            .insert(
2321                module_id.to_string(),
2322                OpenSwap {
2323                    candidate_nonce,
2324                    incumbent_nonce,
2325                    candidate_admitted: false,
2326                },
2327            );
2328    }
2329
2330    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2331    /// the module's recorded one.
2332    pub(crate) fn close_swap(&self, module_id: &str) {
2333        self.swaps
2334            .lock()
2335            .unwrap_or_else(|poisoned| poisoned.into_inner())
2336            .remove(module_id);
2337    }
2338
2339    /// Install the observer told about swap promotions, replacing any earlier
2340    /// one.
2341    pub(crate) fn set_swap_promotion_observer(
2342        &self,
2343        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2344    ) {
2345        *self
2346            .promotion_observer
2347            .0
2348            .lock()
2349            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2350    }
2351
2352    /// Tell the installed observer, if it is still alive, that a swap promoted
2353    /// `registration`.
2354    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2355        let observer = self
2356            .promotion_observer
2357            .0
2358            .lock()
2359            .unwrap_or_else(|poisoned| poisoned.into_inner())
2360            .as_ref()
2361            .and_then(std::sync::Weak::upgrade);
2362        if let Some(observer) = observer {
2363            observer.swap_promoted(registration);
2364        }
2365    }
2366
2367    /// Whether a swap is open for `module_id`.
2368    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2369        self.swaps
2370            .lock()
2371            .unwrap_or_else(|poisoned| poisoned.into_inner())
2372            .contains_key(module_id)
2373    }
2374
2375    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2376    /// respawn would, once cutover has made the candidate the module's process.
2377    /// The swap stays open so the incumbent's nonce keeps attesting until the
2378    /// incumbent has drained and exited.
2379    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2380        let candidate_nonce = self
2381            .swaps
2382            .lock()
2383            .unwrap_or_else(|poisoned| poisoned.into_inner())
2384            .get(module_id)
2385            .map(|swap| swap.candidate_nonce.clone());
2386        let Some(nonce) = candidate_nonce else {
2387            return;
2388        };
2389        self.set_spawn_nonce(module_id, nonce.clone());
2390        if reserved {
2391            self.set_reserved_nonce(module_id, nonce);
2392        }
2393    }
2394
2395    /// The swap gate for a HELLO claiming `module_id`.
2396    ///
2397    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2398    /// presents the candidate nonce, which the reserved gate (holding the
2399    /// incumbent's nonce) would refuse as `reserved_module` before swap
2400    /// admission was ever reached. And it applies to unreserved ids too: for an
2401    /// unreserved id the only thing that ever stopped a second process claiming
2402    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2403    /// refusal a swap lifts for its candidate.
2404    ///
2405    /// The incumbent's own nonce falls through to the ordinary gates, which
2406    /// treat it as they always have (a live incumbent is refused as a
2407    /// duplicate). Anything else while a swap is open is refused, including an
2408    /// absent nonce.
2409    pub(crate) fn swap_hello_admission(
2410        &self,
2411        module_id: &str,
2412        presented: Option<&str>,
2413    ) -> SwapHelloAdmission {
2414        let swaps = self
2415            .swaps
2416            .lock()
2417            .unwrap_or_else(|poisoned| poisoned.into_inner());
2418        let Some(swap) = swaps.get(module_id) else {
2419            return SwapHelloAdmission::NotSwapping;
2420        };
2421        let Some(presented) = presented else {
2422            return SwapHelloAdmission::Refused;
2423        };
2424        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2425            return if swap.candidate_admitted {
2426                SwapHelloAdmission::Refused
2427            } else {
2428                SwapHelloAdmission::Candidate
2429            };
2430        }
2431        if swap
2432            .incumbent_nonce
2433            .as_deref()
2434            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2435        {
2436            return SwapHelloAdmission::NotSwapping;
2437        }
2438        SwapHelloAdmission::Refused
2439    }
2440
2441    /// Record that the swap token has registered a candidate, so it admits no
2442    /// second HELLO.
2443    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2444        if let Some(swap) = self
2445            .swaps
2446            .lock()
2447            .unwrap_or_else(|poisoned| poisoned.into_inner())
2448            .get_mut(module_id)
2449        {
2450            swap.candidate_admitted = true;
2451        }
2452    }
2453
2454    /// Test/support lookup for the current launch nonce of a supervised spawn.
2455    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2456        self.spawn_nonces
2457            .lock()
2458            .unwrap_or_else(|poisoned| poisoned.into_inner())
2459            .get(module_id)
2460            .cloned()
2461    }
2462
2463    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2464    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2465        self.reserved_nonces
2466            .lock()
2467            .unwrap_or_else(|poisoned| poisoned.into_inner())
2468            .get(module_id)
2469            .cloned()
2470            .flatten()
2471    }
2472
2473    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2474        // Normally already marked before the process was spawned; marking here
2475        // too keeps `configured_ids` a superset of the roster for any caller
2476        // that inserts a module directly.
2477        self.mark_configured(module.module_id());
2478        let mut modules = self
2479            .modules
2480            .lock()
2481            .unwrap_or_else(|poisoned| poisoned.into_inner());
2482        modules.insert(module.module_id().to_string(), module)
2483    }
2484
2485    /// Record that the supervisor has taken on `module_id`. Called before the
2486    /// module's first process is spawned, so that by the time that process can
2487    /// register, [`Self::is_configured`] already answers true.
2488    fn mark_configured(&self, module_id: &str) {
2489        self.configured_ids
2490            .lock()
2491            .unwrap_or_else(|poisoned| poisoned.into_inner())
2492            .insert(module_id.to_string());
2493    }
2494
2495    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2496    /// before it was ever put on the roster. A module already on the roster
2497    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2498    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2499        let modules = self
2500            .modules
2501            .lock()
2502            .unwrap_or_else(|poisoned| poisoned.into_inner());
2503        if !modules.contains_key(module_id) {
2504            self.configured_ids
2505                .lock()
2506                .unwrap_or_else(|poisoned| poisoned.into_inner())
2507                .remove(module_id);
2508        }
2509    }
2510
2511    /// Whether `module_id` is a module this daemon supervises: on the roster,
2512    /// or about to be (its process is being spawned right now).
2513    ///
2514    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2515    /// for scopes: a supervised module's process can register and sync before
2516    /// [`Self::get`] can return it, and in that window it is still a module
2517    /// that will sync, not one that never will.
2518    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2519        self.configured_ids
2520            .lock()
2521            .unwrap_or_else(|poisoned| poisoned.into_inner())
2522            .contains(module_id)
2523    }
2524
2525    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2526        let modules = self
2527            .modules
2528            .lock()
2529            .unwrap_or_else(|poisoned| poisoned.into_inner());
2530        modules.get(module_id).cloned()
2531    }
2532
2533    pub(crate) fn record_late_health_answer(
2534        &self,
2535        module_id: &str,
2536        latency_ms: u64,
2537    ) -> Result<bool, SuperviseError> {
2538        let Some(module) = self.get(module_id) else {
2539            return Ok(false);
2540        };
2541        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2542            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2543            state.health.last_late_answer_latency_ms = Some(latency_ms);
2544            // A late answer is an answer: the module served the probe, just past
2545            // the deadline. Leaving the miss streak in place while logging
2546            // "proves the module is alive" is how a CPU-starved module that
2547            // answers every probe a few seconds late still marches to the
2548            // threshold and gets killed — the exact kill class `NoAnswer` is
2549            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2550            // is degradation, and degradation reports; it does not restart.
2551            state.health.consecutive_failures = 0;
2552        })?;
2553        Ok(true)
2554    }
2555
2556    /// Arm the one-shot marker for the module process that this caller
2557    /// deliberately initiated severance against. Generic connection teardown
2558    /// must not call this:
2559    /// a surviving process would otherwise retain an exemption for a later
2560    /// genuine crash.
2561    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2562        let Some(module) = self.get(module_id) else {
2563            return Ok(false);
2564        };
2565        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2566        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2567            return Ok(false);
2568        };
2569        drop(snapshot);
2570        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2571    }
2572
2573    pub fn list(&self) -> Vec<SupervisedModule> {
2574        let modules = self
2575            .modules
2576            .lock()
2577            .unwrap_or_else(|poisoned| poisoned.into_inner());
2578        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2579        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2580        modules
2581    }
2582
2583    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2584        self.activity_images
2585            .lock()
2586            .unwrap_or_else(|p| p.into_inner())
2587            .remove(module_id);
2588        self.spawn_nonces
2589            .lock()
2590            .unwrap_or_else(|poisoned| poisoned.into_inner())
2591            .remove(module_id);
2592        self.close_swap(module_id);
2593        let mut reserved_nonces = self
2594            .reserved_nonces
2595            .lock()
2596            .unwrap_or_else(|poisoned| poisoned.into_inner());
2597        if reserved_nonces.contains_key(module_id) {
2598            // The old nonce must die with the removed process, but the exact-id
2599            // gate remains until an operator explicitly releases it.
2600            reserved_nonces.insert(module_id.to_string(), None);
2601        }
2602        drop(reserved_nonces);
2603        self.reserved_prefix_owners
2604            .lock()
2605            .unwrap_or_else(|poisoned| poisoned.into_inner())
2606            .retain(|_, owner| owner != module_id);
2607        let removed = self
2608            .modules
2609            .lock()
2610            .unwrap_or_else(|poisoned| poisoned.into_inner())
2611            .remove(module_id);
2612        self.configured_ids
2613            .lock()
2614            .unwrap_or_else(|poisoned| poisoned.into_inner())
2615            .remove(module_id);
2616        removed
2617    }
2618
2619    /// Remember a module removed by a non-preview rescan so route.open can
2620    /// distinguish that intentional removal from an unknown id.
2621    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2622        self.removal_tombstones
2623            .lock()
2624            .unwrap_or_else(|poisoned| poisoned.into_inner())
2625            .insert(module_id.to_string(), unix_ms_now());
2626    }
2627
2628    /// Return how long ago a rescan removed this module in milliseconds.
2629    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2630        self.removal_tombstones
2631            .lock()
2632            .unwrap_or_else(|poisoned| poisoned.into_inner())
2633            .get(module_id)
2634            .copied()
2635            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2636    }
2637
2638    /// Retire a reserved-id gate only after its module has left supervision.
2639    ///
2640    /// A retained gate has no live nonce (`None`), so releasing any other entry
2641    /// would weaken a currently configured or otherwise active reservation.
2642    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2643        if self.get(module_id).is_some() {
2644            return false;
2645        }
2646        let mut reserved_nonces = self
2647            .reserved_nonces
2648            .lock()
2649            .unwrap_or_else(|poisoned| poisoned.into_inner());
2650        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2651            return false;
2652        }
2653        reserved_nonces.remove(module_id);
2654        true
2655    }
2656
2657    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2658        Arc::clone(&self.operation_lock)
2659    }
2660}
2661
2662/// Process supervisor for subc-owned singleton modules.
2663#[derive(Debug, Clone)]
2664pub struct Supervisor {
2665    registry: Arc<Registry>,
2666    restart_policy: RestartPolicy,
2667    drain_timeout: Duration,
2668    connection_file_path: Option<PathBuf>,
2669    capture_logs_dir: Option<PathBuf>,
2670    forwarding: Option<Arc<ForwardingTable>>,
2671    process_liveness: Arc<SupervisorProcessLiveness>,
2672    supervisor_handle: Option<SupervisorHandle>,
2673    health: HealthConfig,
2674    daemon_start_clock: crate::clock::StartClock,
2675    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2676    spawn_events: SpawnEventFeed,
2677    provenance_probe: ExecutableIdentityProbe,
2678    /// Every process spawned through this supervisor (and its clones) and not
2679    /// yet reaped, so daemon shutdown can end them.
2680    child_roster: ChildRoster,
2681    #[cfg(target_os = "linux")]
2682    cgroup_placement: Option<subc_cgroup::Placement>,
2683    #[cfg(test)]
2684    test_after_first_spawn: AfterFirstSpawnHook,
2685}
2686
2687/// A roster entry whose monitor and first process have not started yet.
2688/// Keeping the command receiver here lets discovery serve status immediately
2689/// while start/stop requests wait until bootstrap has published the connection file.
2690pub(crate) struct RegisteredConfiguredModule {
2691    spec: ModuleSpec,
2692    runtime: SupervisorRuntimeConfig,
2693    module: SupervisedModule,
2694    commands: mpsc::Receiver<SupervisorCommand>,
2695}
2696
2697/// Test-only hook run on the path that takes on a new module, right after its
2698/// first `spawn_child` returns (the process exists and could already be
2699/// registering) and before that process is handed to the module's supervise
2700/// loop and put on the roster. Lets a test observe what a fast child would see
2701/// in that window without racing a real one.
2702#[cfg(test)]
2703#[derive(Clone, Default)]
2704struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2705
2706#[cfg(test)]
2707type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2708
2709#[cfg(test)]
2710impl fmt::Debug for AfterFirstSpawnHook {
2711    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2712        f.write_str("AfterFirstSpawnHook")
2713    }
2714}
2715
2716#[cfg(test)]
2717impl AfterFirstSpawnHook {
2718    fn run(&self, module_id: &str) {
2719        if let Some(hook) = &self.0 {
2720            hook(module_id);
2721        }
2722    }
2723}
2724
2725impl Supervisor {
2726    #[cfg(test)]
2727    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2728        let supervisor = Self::new(registry, policy);
2729        #[cfg(target_os = "macos")]
2730        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2731        supervisor
2732    }
2733    /// Verify the trampoline once when configured. Missing private OS support
2734    /// refuses every macOS launch by name but does not stop the daemon's control
2735    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2736    /// entry point; the library must not exec an arbitrary hosting program.
2737    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2738        let path = path.into();
2739        #[cfg(target_os = "macos")]
2740        {
2741            let result = probe_privacy_trampoline(&path).map(|()| path);
2742            if let Err(cause) = &result {
2743                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2744            }
2745            self.child_roster.set_privacy_trampoline(result);
2746        }
2747        #[cfg(not(target_os = "macos"))]
2748        let _ = path;
2749        self
2750    }
2751    /// The first step of an announced daemon shutdown, before the notice and
2752    /// before any connection is closed.
2753    ///
2754    /// Sets the daemon-shutdown flag first: from here on no module is
2755    /// respawned (crash restart, operator restart, or swap), and every child
2756    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2757    /// the module exits on the EOF this shutdown gives it or is signalled by a
2758    /// service manager that kills the whole cgroup. Then writes the journal's
2759    /// shutdown marker, which records the instant and closes this daemon
2760    /// incarnation's stretch of the journal.
2761    #[cfg(unix)]
2762    pub(crate) fn begin_daemon_shutdown(&self) {
2763        self.child_roster.close();
2764        if let Some(journal) = &self.terminal_journal {
2765            journal.stamp_shutdown();
2766        }
2767    }
2768
2769    /// Announce a cut while established connections can still carry replies.
2770    /// These budgets promise notice and a bounded wait, not child completion;
2771    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2772    #[cfg(unix)]
2773    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2774        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2775        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2776        let Some(forwarding) = &self.forwarding else {
2777            return Ok(());
2778        };
2779        let modules = forwarding
2780            .begin_daemon_drain()
2781            .map_err(SuperviseError::Forwarding)?;
2782        let deadline_ms =
2783            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2784        let mut notices = tokio::task::JoinSet::new();
2785        let mut drains = Vec::new();
2786        for (module_id, endpoint) in modules {
2787            let Some(target) = forwarding
2788                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2789                .map_err(SuperviseError::Forwarding)?
2790            else {
2791                continue;
2792            };
2793            let routes = forwarding
2794                .endpoint_routes(target.endpoint)
2795                .map_err(SuperviseError::Forwarding)?;
2796            // Restart allows deployed consumers to reopen after the new daemon
2797            // appears. The wire reason stays `restart`; what tells a daemon cut
2798            // apart from a module restart afterwards is the terminal record
2799            // itself, whose disposition is `daemon_shutdown` for every exit
2800            // observed once `begin_daemon_shutdown` has run.
2801            let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2802                reason: RouteCloseReason::Restart,
2803                deadline_ms,
2804            })
2805            .expect("module draining serializes");
2806            let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2807            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2808            for route in routes {
2809                let client = route.goodbye_target;
2810                if let Some((_, channels)) = clients
2811                    .iter_mut()
2812                    .find(|(existing, _)| existing.connection_id == client.connection_id)
2813                {
2814                    channels.push(client.channel);
2815                } else {
2816                    let channel = client.channel;
2817                    clients.push((client, vec![channel]));
2818                }
2819            }
2820            for (client, mut channels) in clients {
2821                channels.sort_unstable();
2822                channels.dedup();
2823                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2824                    module_id: module_id.clone(),
2825                    channels,
2826                    reason: RouteCloseReason::Restart,
2827                })
2828                .expect("route closing serializes");
2829                recipients.push((client.sink, client.negotiated_ver, closing));
2830            }
2831            for (sink, version, body) in recipients {
2832                notices.spawn(async move {
2833                    let frame = Frame::build_with_version(
2834                        version,
2835                        FrameType::Push,
2836                        control_flags(),
2837                        0,
2838                        0,
2839                        0,
2840                        body,
2841                    )
2842                    .expect("bounded lifecycle notice frame builds");
2843                    sink.send_flushed(frame).await
2844                });
2845            }
2846            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2847            drains.push((module_id, target.endpoint, gauges));
2848        }
2849        // A quiet forwarding table is not proof that queued notices reached the
2850        // socket. Wait for writer flush acknowledgements before testing quiescence.
2851        let notice_deadline = Instant::now() + NOTICE_BUDGET;
2852        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2853            if !matches!(result, Ok(Ok(()))) {
2854                warn!(?result, "daemon shutdown notice delivery failed");
2855            }
2856        }
2857        notices.abort_all();
2858        let deadline = Instant::now() + DRAIN_BUDGET;
2859        let mut waits = tokio::task::JoinSet::new();
2860        for (module_id, endpoint, gauges) in drains {
2861            let forwarding = Arc::clone(forwarding);
2862            let mut runtime = self.runtime_config();
2863            runtime.health.cadence = Duration::from_millis(100);
2864            waits.spawn(async move {
2865                let wait_result = wait_for_forwarding_quiescence(
2866                    &forwarding,
2867                    &module_id,
2868                    &runtime,
2869                    endpoint,
2870                    deadline,
2871                    &gauges,
2872                    DrainScope::Endpoint(endpoint),
2873                )
2874                .await;
2875                // Store the measurement before the task returns: quiescence
2876                // may be reached just as the deadline stops the join loop from
2877                // collecting results. Unfinished or failed waits report false.
2878                forwarding
2879                    .record_daemon_drain_result(
2880                        endpoint,
2881                        drained_after_quiescence_wait(&wait_result),
2882                    )
2883                    .map_err(SuperviseError::Forwarding)?;
2884                wait_result
2885            });
2886        }
2887        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2888            if !matches!(result, Ok(Ok(true))) {
2889                warn!(?result, "daemon shutdown drain did not reach quiescence");
2890            }
2891        }
2892        Ok(())
2893    }
2894
2895    /// The last step of an announced daemon shutdown, after the notice and the
2896    /// drain: send every registered module a module GOODBYE, the same planned
2897    /// stop signal `ck module stop` gives, then close every connection so each
2898    /// subc module sees EOF and starts its own teardown, then end every
2899    /// supervised child that has not exited
2900    /// by its own deadline (its drain budget, capped). Modules lead their own
2901    /// process groups, so a
2902    /// service manager's group kill no longer reaches them; without this a
2903    /// child that does not stop on EOF (every `protocol: "none"` child, which
2904    /// has no connection) would outlive the daemon. Every wait is bounded (see
2905    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
2906    #[cfg(unix)]
2907    pub(crate) async fn end_children_for_daemon_shutdown(
2908        &self,
2909        already_escalated: bool,
2910        escalate: impl std::future::Future<Output = ()>,
2911    ) {
2912        tokio::pin!(escalate);
2913        let mut escalated = already_escalated;
2914        if let Some(forwarding) = &self.forwarding {
2915            let reason = CloseReason::new(
2916                "daemon_shutdown",
2917                "the daemon is exiting after its shutdown notice and drain",
2918            );
2919            if escalated {
2920                // The operator asked to stop waiting: queue the GOODBYEs but
2921                // do not wait for them to be written.
2922                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2923            } else {
2924                tokio::select! {
2925                    biased;
2926                    _ = escalate.as_mut() => {
2927                        info!("second SIGTERM: abandoning module GOODBYE delivery");
2928                        escalated = true;
2929                    }
2930                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2931                }
2932            }
2933            let closed = forwarding.close_all_connections(&reason);
2934            debug!(closed, "closed established connections for daemon shutdown");
2935        }
2936        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
2937        // already completed and must not be polled again; the child shutdown
2938        // wait is told it is escalated and gets a future that never fires.
2939        let escalated_here = escalated && !already_escalated;
2940        let remaining_escalate = async move {
2941            if escalated_here {
2942                std::future::pending::<()>().await;
2943            } else {
2944                escalate.await;
2945            }
2946        };
2947        crate::child_roster::end_children_for_daemon_shutdown(
2948            &self.child_roster,
2949            escalated,
2950            remaining_escalate,
2951        )
2952        .await;
2953    }
2954
2955    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2956        Self {
2957            registry,
2958            restart_policy,
2959            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2960            connection_file_path: None,
2961            capture_logs_dir: None,
2962            forwarding: None,
2963            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2964            supervisor_handle: None,
2965            health: HealthConfig::default(),
2966            daemon_start_clock: crate::clock::StartClock::capture(),
2967            terminal_journal: None,
2968            spawn_events: SpawnEventFeed::default(),
2969            provenance_probe: ExecutableIdentityProbe::default(),
2970            child_roster: ChildRoster::default(),
2971            #[cfg(target_os = "linux")]
2972            cgroup_placement: None,
2973            #[cfg(test)]
2974            test_after_first_spawn: AfterFirstSpawnHook::default(),
2975        }
2976    }
2977
2978    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
2979        self.drain_timeout = drain_timeout;
2980        self
2981    }
2982
2983    pub fn with_process_liveness(
2984        mut self,
2985        process_liveness: Arc<SupervisorProcessLiveness>,
2986    ) -> Self {
2987        self.process_liveness = process_liveness;
2988        self
2989    }
2990
2991    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
2992        self.connection_file_path = Some(connection_file_path.into());
2993        self
2994    }
2995
2996    /// Enables daemon-owned capture files for supervised stdout and stderr.
2997    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
2998        self.capture_logs_dir = Some(logs_dir.into());
2999        self
3000    }
3001
3002    /// Names this daemon lifetime in spawn events, independently of whether a
3003    /// terminal journal is configured.
3004    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3005        // A millisecond start stamp can repeat after clock rollback or a rapid
3006        // restart. Use the connection file's random daemon_id instead: it already
3007        // identifies this daemon lifetime independently of the wall clock.
3008        self.spawn_events.configure_incarnation(daemon_incarnation);
3009        self
3010    }
3011
3012    /// Enables best-effort history shared by every supervised module. Without
3013    /// it, terminal history is kept only in each module's in-memory ring.
3014    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3015        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3016        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3017            path,
3018            daemon_incarnation,
3019        )));
3020        this
3021    }
3022
3023    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3024        self.forwarding = Some(forwarding);
3025        self
3026    }
3027
3028    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3029        self.spawn_events = supervisor_handle.spawn_events.clone();
3030        self.supervisor_handle = Some(supervisor_handle);
3031        self
3032    }
3033
3034    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3035        self.health = health;
3036        self
3037    }
3038
3039    /// Keeps a record of every live child at `path`, rewritten on each spawn and
3040    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
3041    /// record is kept.
3042    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3043        self.child_roster.record_to(path.into());
3044        self
3045    }
3046
3047    #[cfg(target_os = "linux")]
3048    pub fn with_cgroup_placement(
3049        mut self,
3050        cgroup_placement: Option<subc_cgroup::Placement>,
3051    ) -> Self {
3052        self.cgroup_placement = cgroup_placement;
3053        self
3054    }
3055
3056    /// Spawn `spec.program` and start monitoring it.
3057    ///
3058    /// The child is expected to parse `--subc <connection-file-path>`, read the
3059    /// TCP+key connection file, authenticate to the already-running listener, and
3060    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
3061    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3062        validate_spec(&spec)?;
3063        self.establish_identity(&spec);
3064
3065        let runtime = self.runtime_config();
3066        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3067        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3068        let spawned = spawn_child(
3069            &spec,
3070            runtime.connection_file_path.as_deref(),
3071            self.supervisor_handle.as_ref(),
3072            &runtime.stderr_ring,
3073            runtime.capture_logs_dir.as_deref(),
3074            &runtime.child_roster,
3075            #[cfg(target_os = "linux")]
3076            runtime.cgroup_placement.as_ref(),
3077        );
3078        #[cfg(test)]
3079        self.test_after_first_spawn.run(&spec.module_id);
3080        let child = match spawned {
3081            Ok(child) => child,
3082            Err(err) => {
3083                // Unlike the configured paths, a failed `spawn` leaves nothing
3084                // on the roster, so the module must not stay marked configured.
3085                self.abandon_unrostered(&spec.module_id);
3086                return Err(err);
3087            }
3088        };
3089        self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3090        if let Some(spawn) = activity_spawn {
3091            spawn.complete(&snapshot);
3092        }
3093
3094        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3095    }
3096
3097    /// Make `spec`'s module count as configured, with its identity gates
3098    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3099    /// exists.
3100    ///
3101    /// Every path that takes on a new module calls this before `spawn_child`.
3102    /// The order is the point: the child can connect, register, sync its
3103    /// scopes and ask about them as soon as it is spawned, and the module is
3104    /// only put on the roster after `spawn_child` returns. Were the mark set
3105    /// with the roster entry, a fast child would see its own owner reported
3106    /// as not configured, and a scoped `route.open` in that window would be
3107    /// refused as terminal `scope_not_live` ("will never sync") instead of
3108    /// retryable `scope_not_synced`.
3109    fn establish_identity(&self, spec: &ModuleSpec) {
3110        if let Some(supervisor_handle) = &self.supervisor_handle {
3111            supervisor_handle.apply_identity_configuration(spec);
3112            supervisor_handle.mark_configured(&spec.module_id);
3113        }
3114    }
3115
3116    /// Take back [`Self::establish_identity`]'s configured mark when the
3117    /// module will not be put on the roster after all.
3118    fn abandon_unrostered(&self, module_id: &str) {
3119        if let Some(supervisor_handle) = &self.supervisor_handle {
3120            supervisor_handle.unmark_configured_unless_rostered(module_id);
3121        }
3122    }
3123
3124    /// Record a freshly spawned first process as running. On failure the
3125    /// module never reaches the roster, so its configured mark is taken back.
3126    fn mark_first_process_running(
3127        &self,
3128        spec: &ModuleSpec,
3129        runtime: &SupervisorRuntimeConfig,
3130        snapshot: &SharedSnapshot,
3131        child: &SupervisedChild,
3132    ) -> Result<(), SuperviseError> {
3133        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3134            self.abandon_unrostered(&spec.module_id);
3135            return Err(err);
3136        }
3137        self.process_liveness
3138            .track(spec.module_id.clone(), Arc::clone(snapshot));
3139        Ok(())
3140    }
3141
3142    /// Start supervising a module declared in daemon configuration.
3143    ///
3144    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3145    /// failures in the supervisor handle so operator-facing `supervisor.list`
3146    /// reflects every configured module while daemon startup continues.
3147    pub fn supervise_configured(
3148        &self,
3149        spec: ModuleSpec,
3150        enabled: bool,
3151    ) -> Result<SupervisedModule, SuperviseError> {
3152        self.supervise_configured_requested(
3153            spec,
3154            enabled,
3155            self.health.clone(),
3156            None,
3157            self.restart_policy,
3158            None,
3159        )
3160    }
3161
3162    /// Supervise a configured module with its own health, drain, and crash
3163    /// budget. The restart policy is per-module because the config file is:
3164    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3165    /// module that is expensive to restart should not be forced onto the same
3166    /// budget as one that is cheap.
3167    pub fn supervise_configured_with_health(
3168        &self,
3169        spec: ModuleSpec,
3170        enabled: bool,
3171        health: HealthConfig,
3172        drain_timeout_ms: Option<u64>,
3173        restart_policy: RestartPolicy,
3174    ) -> Result<SupervisedModule, SuperviseError> {
3175        self.supervise_configured_requested(
3176            spec,
3177            enabled,
3178            health,
3179            drain_timeout_ms,
3180            restart_policy,
3181            None,
3182        )
3183    }
3184
3185    pub(crate) fn supervise_configured_requested(
3186        &self,
3187        spec: ModuleSpec,
3188        enabled: bool,
3189        health: HealthConfig,
3190        drain_timeout_ms: Option<u64>,
3191        restart_policy: RestartPolicy,
3192        requester: Option<Principal>,
3193    ) -> Result<SupervisedModule, SuperviseError> {
3194        let registered = self.register_configured_with_health(
3195            spec,
3196            enabled,
3197            health,
3198            drain_timeout_ms,
3199            restart_policy,
3200        )?;
3201        self.start_registered_configured(registered, requester)
3202    }
3203
3204    /// Record a configured module as starting or disabled, without launching a
3205    /// process or running its monitor. Bootstrap registers the entire roster
3206    /// before publishing discovery, then activates it once children can read
3207    /// the connection file passed in their `--subc` argument.
3208    pub(crate) fn register_configured_with_health(
3209        &self,
3210        spec: ModuleSpec,
3211        enabled: bool,
3212        health: HealthConfig,
3213        drain_timeout_ms: Option<u64>,
3214        restart_policy: RestartPolicy,
3215    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3216        validate_spec(&spec)?;
3217        self.establish_identity(&spec);
3218
3219        let mut runtime = self.runtime_config();
3220        runtime.health = health;
3221        runtime.restart_policy = restart_policy;
3222        if let Some(ms) = drain_timeout_ms {
3223            runtime.drain_timeout = Duration::from_millis(ms);
3224            *runtime
3225                .effective_drain_timeout
3226                .lock()
3227                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3228        }
3229        let snapshot = if enabled {
3230            SupervisorSnapshot::starting()
3231        } else {
3232            SupervisorSnapshot::disabled()
3233        };
3234        let snapshot = {
3235            let mut snapshot = snapshot;
3236            if enabled {
3237                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3238                {
3239                    snapshot.waiting_for_previous_pid = Some(pid);
3240                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3241                }
3242            }
3243            snapshot
3244        };
3245        let (module, commands) =
3246            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3247        Ok(RegisteredConfiguredModule {
3248            spec,
3249            runtime,
3250            module,
3251            commands,
3252        })
3253    }
3254
3255    /// Activate a previously registered module. Queued control requests are
3256    /// served by the monitor after the initial configured spawn has settled.
3257    pub(crate) fn start_registered_configured(
3258        &self,
3259        registered: RegisteredConfiguredModule,
3260        requester: Option<Principal>,
3261    ) -> Result<SupervisedModule, SuperviseError> {
3262        let RegisteredConfiguredModule {
3263            spec,
3264            mut runtime,
3265            module,
3266            commands,
3267        } = registered;
3268        runtime.start_requester = requester;
3269        let snapshot = &module.inner.snapshot;
3270        let should_spawn = {
3271            let state = lock_snapshot(snapshot)?;
3272            state.enabled && state.waiting_for_previous_pid.is_none()
3273        };
3274        let child = if should_spawn {
3275            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3276            let spawned = spawn_child(
3277                &spec,
3278                runtime.connection_file_path.as_deref(),
3279                self.supervisor_handle.as_ref(),
3280                &runtime.stderr_ring,
3281                runtime.capture_logs_dir.as_deref(),
3282                &runtime.child_roster,
3283                #[cfg(target_os = "linux")]
3284                runtime.cgroup_placement.as_ref(),
3285            );
3286            #[cfg(test)]
3287            self.test_after_first_spawn.run(&spec.module_id);
3288            match spawned {
3289                Ok(child) => {
3290                    self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3291                    if let Some(spawn) = activity_spawn {
3292                        spawn.complete(snapshot);
3293                    }
3294                    Some(child)
3295                }
3296                Err(err) => {
3297                    if runtime.health.critical {
3298                        error!(
3299                            module_id = %spec.module_id,
3300                            program = %spec.program.display(),
3301                            error = %err,
3302                            "critical configured module failed to spawn; marking failed and alerting"
3303                        );
3304                    } else {
3305                        error!(
3306                            module_id = %spec.module_id,
3307                            program = %spec.program.display(),
3308                            error = %err,
3309                            "configured module failed to spawn; marking failed and continuing"
3310                        );
3311                    }
3312                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
3313                        state.state = ModuleState::Failed;
3314                        clear_current_process_facts(state);
3315                    })?;
3316                    None
3317                }
3318            }
3319        } else {
3320            None
3321        };
3322        self.start_monitor(spec, runtime, &module, child, commands);
3323        Ok(module)
3324    }
3325
3326    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3327        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3328        SupervisorRuntimeConfig {
3329            start_requester: None,
3330            pending_activity_restart: false,
3331            provenance_probe: self.provenance_probe.clone(),
3332            scheduled_respawn: Arc::default(),
3333            deferred_reload_reply: Arc::default(),
3334            restart_policy: self.restart_policy,
3335            drain_timeout: self.drain_timeout,
3336            // Shared with this module's roster copy: daemon shutdown waits on
3337            // each child for the module's own drain budget, as resolved now.
3338            child_roster: self
3339                .child_roster
3340                .for_module(Arc::clone(&effective_drain_timeout)),
3341            effective_drain_timeout,
3342            default_drain_timeout: self.drain_timeout,
3343            health: self.health.clone(),
3344            connection_file_path: self.connection_file_path.clone(),
3345            capture_logs_dir: self.capture_logs_dir.clone(),
3346            forwarding: self.forwarding.clone(),
3347            supervisor_handle: self.supervisor_handle.clone(),
3348            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3349            terminal_ring: Arc::new(Mutex::new(
3350                TerminalRing::new(
3351                    TerminalRingConfig::default(),
3352                    self.daemon_start_clock.started_at_ms(),
3353                )
3354                .with_start_clock(self.daemon_start_clock)
3355                .with_journal(self.terminal_journal.clone())
3356                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3357            )),
3358            spawn_events: self.spawn_events.clone(),
3359            #[cfg(target_os = "linux")]
3360            cgroup_placement: self.cgroup_placement.clone(),
3361            #[cfg(test)]
3362            test_seed_stale_facts_before_enable_spawn: false,
3363            #[cfg(test)]
3364            test_reload_exit_record_gate: None,
3365        }
3366    }
3367
3368    fn supervised_module(
3369        &self,
3370        spec: ModuleSpec,
3371        runtime: SupervisorRuntimeConfig,
3372        snapshot: SharedSnapshot,
3373        child: Option<SupervisedChild>,
3374    ) -> SupervisedModule {
3375        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3376        self.start_monitor(spec, runtime, &module, child, commands);
3377        module
3378    }
3379
3380    fn register_module(
3381        &self,
3382        spec: &ModuleSpec,
3383        runtime: &SupervisorRuntimeConfig,
3384        snapshot: SharedSnapshot,
3385    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3386        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3387            spec: spec.clone(),
3388            health: runtime.health.clone(),
3389        }));
3390        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3391        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3392        // The module's OWN policy, which may be its per-module config rather than
3393        // the supervisor-wide one; status must report the budget the supervise
3394        // loop actually enforces.
3395        let restart_policy = runtime.restart_policy;
3396        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3397        let (tx, rx) = mpsc::channel(4);
3398        let module_id = spec.module_id.clone();
3399        let module = SupervisedModule {
3400            inner: Arc::new(SupervisedModuleInner {
3401                module_id: module_id.clone(),
3402                registry: Arc::clone(&self.registry),
3403                snapshot,
3404                configuration,
3405                stderr_ring,
3406                terminal_ring,
3407                commands: tx,
3408                monitor: Mutex::new(None),
3409                restart_policy,
3410                effective_drain_timeout,
3411                provenance_probe: self.provenance_probe.clone(),
3412            }),
3413        };
3414        // Identity gates and the configured mark must precede the roster entry,
3415        // so a client never sees a module without its admission policy.
3416        if let Some(supervisor_handle) = &self.supervisor_handle {
3417            supervisor_handle.insert(module.clone());
3418        }
3419        (module, rx)
3420    }
3421
3422    fn start_monitor(
3423        &self,
3424        spec: ModuleSpec,
3425        runtime: SupervisorRuntimeConfig,
3426        module: &SupervisedModule,
3427        child: Option<SupervisedChild>,
3428        commands: mpsc::Receiver<SupervisorCommand>,
3429    ) {
3430        let monitor = tokio::spawn(supervise_loop(
3431            spec,
3432            runtime,
3433            Arc::clone(&self.registry),
3434            Arc::clone(&self.process_liveness),
3435            Arc::clone(&module.inner.snapshot),
3436            child,
3437            commands,
3438        ));
3439        *module
3440            .inner
3441            .monitor
3442            .lock()
3443            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3444    }
3445}
3446
3447impl Default for Supervisor {
3448    fn default() -> Self {
3449        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3450    }
3451}
3452
3453/// Handle to one supervised child process.
3454#[derive(Clone)]
3455pub struct SupervisedModule {
3456    inner: Arc<SupervisedModuleInner>,
3457}
3458
3459struct SupervisedModuleInner {
3460    module_id: String,
3461    registry: Arc<Registry>,
3462    snapshot: SharedSnapshot,
3463    configuration: Arc<Mutex<SupervisedConfiguration>>,
3464    stderr_ring: Arc<Mutex<StderrRing>>,
3465    terminal_ring: Arc<Mutex<TerminalRing>>,
3466    commands: mpsc::Sender<SupervisorCommand>,
3467    monitor: Mutex<Option<JoinHandle<()>>>,
3468    /// Copied from the supervisor's runtime config at spawn so `status()` can
3469    /// report the restart budget without reaching back into the supervisor. The
3470    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3471    restart_policy: RestartPolicy,
3472    effective_drain_timeout: Arc<Mutex<Duration>>,
3473    provenance_probe: ExecutableIdentityProbe,
3474}
3475
3476impl fmt::Debug for SupervisedModule {
3477    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3478        f.debug_struct("SupervisedModule")
3479            .field("module_id", &self.inner.module_id)
3480            .field("status", &self.status())
3481            .finish_non_exhaustive()
3482    }
3483}
3484
3485impl SupervisedModule {
3486    pub fn module_id(&self) -> &str {
3487        &self.inner.module_id
3488    }
3489
3490    /// Test-only: put one probe miss on the streak, the way
3491    /// `handle_health_probe_failure` does, so tests can assert what a later
3492    /// event does to the streak without driving the whole probe loop.
3493    #[cfg(test)]
3494    pub(crate) fn record_health_probe_failure_for_test(
3495        &self,
3496        detail: &str,
3497    ) -> Result<(), SuperviseError> {
3498        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3499            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3500            state.health.detail = Some(detail.to_string());
3501        })
3502    }
3503
3504    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3505        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3506    }
3507
3508    pub(crate) fn registration_end_reason(
3509        &self,
3510    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3511        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3512        Ok(match snapshot.state {
3513            ModuleState::Draining if snapshot.draining_to_replace => {
3514                Some(RegistrationEndReason::SupervisorRestart)
3515            }
3516            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3517            ModuleState::Restarting if snapshot.draining_to_replace => {
3518                Some(RegistrationEndReason::SupervisorRestart)
3519            }
3520            _ => None,
3521        })
3522    }
3523
3524    /// The module's retained stderr, newest lines last.
3525    ///
3526    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3527    /// module, `supervisor.list` renders every module, and putting it in the
3528    /// shared snapshot would make each status read carry a payload almost nobody
3529    /// asked for. Callers that want the text ask for it.
3530    pub fn stderr_tail(
3531        &self,
3532        max_lines: Option<usize>,
3533        max_bytes: Option<usize>,
3534    ) -> StderrTailSnapshot {
3535        self.inner
3536            .stderr_ring
3537            .lock()
3538            .unwrap_or_else(|poisoned| poisoned.into_inner())
3539            .snapshot(max_lines, max_bytes)
3540    }
3541
3542    /// The module's bounded terminal history, oldest retained exit first.
3543    ///
3544    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3545    /// daemon whose in-memory history was necessarily reset.
3546    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3547        self.inner
3548            .terminal_ring
3549            .lock()
3550            .unwrap_or_else(|poisoned| poisoned.into_inner())
3551            .snapshot()
3552    }
3553
3554    /// Retained observations from the current ring and all journal generations.
3555    ///
3556    /// Blocking: this reads the journal files. Async callers use
3557    /// [`Self::read_durable_terminal_history`].
3558    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3559        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3560    }
3561
3562    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3563    /// read (up to every retained generation) never occupies a runtime worker.
3564    /// Fails only if the blocking task could not finish (runtime shutdown or a
3565    /// panic in the read).
3566    pub(crate) async fn read_durable_terminal_history(
3567        &self,
3568    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3569        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3570        let module_id = self.inner.module_id.clone();
3571        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3572            .await
3573    }
3574
3575    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3576        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3577            .map(|(status, _)| status)
3578    }
3579
3580    pub(crate) fn record_deliberate_severance(
3581        &self,
3582        identity: ProcessIdentity,
3583    ) -> Result<bool, SuperviseError> {
3584        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3585        if snapshot.pid != Some(identity.pid)
3586            || snapshot.process_start_time != Some(identity.start_time)
3587        {
3588            return Ok(false);
3589        }
3590        snapshot.deliberate_severance = Some(identity);
3591        Ok(true)
3592    }
3593
3594    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3595    ///
3596    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3597    /// does not produce reader-observability logs.
3598    pub(crate) fn status_for_control(
3599        &self,
3600        caller: &'static str,
3601    ) -> Result<ModuleStatus, SuperviseError> {
3602        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3603            .map(|(status, _)| status)
3604    }
3605
3606    fn status_with_snapshot_lock(
3607        &self,
3608        snapshot: &SharedSnapshot,
3609        caller: Option<&'static str>,
3610    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3611        let mut guard = match caller {
3612            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3613            None => lock_snapshot(snapshot)?,
3614        };
3615        // Read the budget through the pruning path so a reader sees the same
3616        // in-window count the restart decision would use, not a stale total.
3617        let restart_count =
3618            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3619        let snapshot = guard.clone();
3620        drop(guard);
3621        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3622            SuperviseError::StatePoisoned {
3623                module_id: Some(self.inner.module_id.clone()),
3624            }
3625        })?;
3626        let registration_active = self
3627            .inner
3628            .registry
3629            .get_module(&self.inner.module_id)
3630            .map_err(SuperviseError::Registry)?
3631            .is_some();
3632        let protocol = snapshot
3633            .spawned_protocol
3634            .unwrap_or(self.declared_protocol()?);
3635        let running_process =
3636            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3637        // Registration is the difference between the two protocols and the only
3638        // one: a subc module that has not registered cannot serve a request even
3639        // though its process is up, and a `none` module never registers at all,
3640        // so requiring it there would pin `live` to false for the whole life of
3641        // a perfectly healthy process.
3642        let live = match protocol {
3643            ModuleProtocol::Subc => running_process && registration_active,
3644            ModuleProtocol::None => running_process,
3645        };
3646
3647        Ok((
3648            ModuleStatus {
3649                waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3650                waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3651                module_id: self.inner.module_id.clone(),
3652                launch_nonce_source: observed_launch_nonce_source(&snapshot),
3653                state: snapshot.state,
3654                enabled: snapshot.enabled,
3655                process_alive: snapshot.process_alive,
3656                registration_active,
3657                protocol,
3658                live,
3659                restart_count,
3660                lifetime_restarts: snapshot.lifetime_restarts,
3661                spawn_generation: snapshot.spawn_generation,
3662                max_restarts: self.inner.restart_policy.max_restarts,
3663                restart_window: self.inner.restart_policy.window,
3664                drain_timeout,
3665                restart_backoff: self.inner.restart_policy.backoff,
3666                restart_max_backoff: self.inner.restart_policy.max_backoff,
3667                pid: snapshot.reported_pid(),
3668                spawned_at_ms: snapshot.spawned_at_ms,
3669                spawned_from: snapshot.spawned_from,
3670                launch_spec_digest: snapshot.launch_spec_digest,
3671                process_start_time: snapshot.process_start_time,
3672                last_exit: snapshot.last_exit,
3673                health: snapshot.health,
3674            },
3675            snapshot.spawned_file_identity.clone(),
3676        ))
3677    }
3678
3679    #[cfg(test)]
3680    pub(crate) fn hold_snapshot_for_test(
3681        &self,
3682        acquired: std::sync::mpsc::Sender<()>,
3683        hold: Duration,
3684    ) -> std::thread::JoinHandle<()> {
3685        let snapshot = Arc::clone(&self.inner.snapshot);
3686        std::thread::spawn(move || {
3687            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3688            acquired
3689                .send(())
3690                .expect("test receiver waits for snapshot lock");
3691            std::thread::sleep(hold);
3692        })
3693    }
3694
3695    /// The status and the running-image check for `supervisor.provenance`,
3696    /// taken from one status read. The exec acknowledgement can land between
3697    /// two separate reads, and the reply would then pair "no pid yet" with an
3698    /// image observed after the module started, which describes no single
3699    /// moment.
3700    pub(crate) async fn status_and_running_image_agreement(
3701        &self,
3702    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3703        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3704        let image = self
3705            .inner
3706            .provenance_probe
3707            .observe(
3708                status.pid,
3709                status.spawned_from.as_deref(),
3710                identity,
3711                status.process_start_time,
3712            )
3713            .await;
3714        Ok((status, image))
3715    }
3716
3717    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3718        let snapshot = match lock_snapshot(&self.inner.snapshot) {
3719            Ok(snapshot) => snapshot.clone(),
3720            Err(_) => {
3721                return subc_control::RunningImageAgreement::Unavailable {
3722                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
3723                };
3724            }
3725        };
3726        self.inner
3727            .provenance_probe
3728            .observe(
3729                snapshot.reported_pid(),
3730                snapshot.spawned_from.as_deref(),
3731                snapshot.spawned_file_identity.clone(),
3732                snapshot.process_start_time,
3733            )
3734            .await
3735    }
3736
3737    /// Memory and CPU time of the module's current process, read now. Only the
3738    /// process the supervisor spawned is read, not processes it has started.
3739    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3740        #[cfg(windows)]
3741        {
3742            let image = lock_snapshot(&self.inner.snapshot)
3743                .ok()
3744                .and_then(|snapshot| {
3745                    snapshot
3746                        .spawned_file_identity
3747                        .as_ref()
3748                        .and_then(|identity| identity.windows_image.clone())
3749                });
3750            match image {
3751                Some(image) => {
3752                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
3753                }
3754                None => subc_control::ChildResourceUsage::Unavailable {
3755                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3756                },
3757            }
3758        }
3759        #[cfg(not(windows))]
3760        {
3761            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3762                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3763                Err(_) => {
3764                    return subc_control::ChildResourceUsage::Unavailable {
3765                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3766                    }
3767                }
3768            };
3769            crate::child_resources::read(pid, start_time)
3770        }
3771    }
3772
3773    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3774        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3775        Ok(match snapshot.state {
3776            ModuleState::Restarting => true,
3777            ModuleState::Failed | ModuleState::Disabled => false,
3778            _ if snapshot.enabled && snapshot.respawn_pending => true,
3779            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3780        })
3781    }
3782
3783    #[cfg(test)]
3784    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3785        self.is_warming_with_snapshot_lock(None)
3786    }
3787
3788    pub(crate) fn is_warming_for_control(
3789        &self,
3790        caller: &'static str,
3791    ) -> Result<bool, SuperviseError> {
3792        self.is_warming_with_snapshot_lock(Some(caller))
3793    }
3794
3795    fn is_warming_with_snapshot_lock(
3796        &self,
3797        caller: Option<&'static str>,
3798    ) -> Result<bool, SuperviseError> {
3799        let snapshot = match caller {
3800            Some(caller) => {
3801                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3802            }
3803            None => lock_snapshot(&self.inner.snapshot)?,
3804        }
3805        .clone();
3806        Ok(matches!(
3807            snapshot.state,
3808            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3809        ))
3810    }
3811
3812    /// Drain the module and stop monitoring it.
3813    pub async fn drain(&self) -> Result<(), SuperviseError> {
3814        self.stop().await
3815    }
3816
3817    pub(crate) async fn retire_requested(
3818        &self,
3819        requester: Option<Principal>,
3820    ) -> Result<(), SuperviseError> {
3821        match self.state()? {
3822            ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3823            ModuleState::Starting
3824            | ModuleState::Running
3825            | ModuleState::Unresponsive
3826            | ModuleState::Restarting
3827            | ModuleState::Draining
3828            | ModuleState::Disabled => {}
3829        }
3830
3831        let (reply_tx, reply_rx) = oneshot::channel();
3832        self.inner
3833            .commands
3834            .send(SupervisorCommand::Retire {
3835                requester,
3836                reply: reply_tx,
3837            })
3838            .await
3839            .map_err(|_| SuperviseError::CommandClosed {
3840                module_id: self.inner.module_id.clone(),
3841            })?;
3842        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3843            module_id: self.inner.module_id.clone(),
3844        })?
3845    }
3846
3847    pub async fn stop(&self) -> Result<(), SuperviseError> {
3848        self.stop_requested(None).await
3849    }
3850
3851    pub(crate) async fn stop_requested(
3852        &self,
3853        requester: Option<Principal>,
3854    ) -> Result<(), SuperviseError> {
3855        match self.state()? {
3856            ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3857            ModuleState::Starting
3858            | ModuleState::Running
3859            | ModuleState::Unresponsive
3860            | ModuleState::Restarting
3861            | ModuleState::Draining
3862            | ModuleState::Disabled => {}
3863        }
3864
3865        let (reply_tx, reply_rx) = oneshot::channel();
3866        self.inner
3867            .commands
3868            .send(SupervisorCommand::Drain {
3869                requester,
3870                reply: reply_tx,
3871            })
3872            .await
3873            .map_err(|_| SuperviseError::CommandClosed {
3874                module_id: self.inner.module_id.clone(),
3875            })?;
3876        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3877            module_id: self.inner.module_id.clone(),
3878        })?
3879    }
3880
3881    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3882        self.restart_requested(drain_timeout_ms, None).await
3883    }
3884
3885    pub(crate) async fn restart_requested(
3886        &self,
3887        drain_timeout_ms: Option<u64>,
3888        requester: Option<Principal>,
3889    ) -> Result<(), SuperviseError> {
3890        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3891        let (reply_tx, reply_rx) = oneshot::channel();
3892        self.inner
3893            .commands
3894            .send(SupervisorCommand::Restart {
3895                requester,
3896                drain_timeout_ms,
3897                received_at_generation,
3898                queued_at: Instant::now(),
3899                reply: reply_tx,
3900            })
3901            .await
3902            .map_err(|_| SuperviseError::CommandClosed {
3903                module_id: self.inner.module_id.clone(),
3904            })?;
3905        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3906            module_id: self.inner.module_id.clone(),
3907        })?
3908    }
3909
3910    /// Blue/green restart: see [`SupervisorCommand::Swap`] and the
3911    /// `supervisor_swap` module. Returns once the swap has cut over (the old
3912    /// process then drains in the background of the supervise loop) or has
3913    /// failed, leaving the old process serving.
3914    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3915        self.swap_requested(ready_timeout, None).await
3916    }
3917
3918    pub(crate) async fn swap_requested(
3919        &self,
3920        ready_timeout: Option<Duration>,
3921        requester: Option<Principal>,
3922    ) -> Result<(), SuperviseError> {
3923        let (reply_tx, reply_rx) = oneshot::channel();
3924        self.inner
3925            .commands
3926            .send(SupervisorCommand::Swap {
3927                requester,
3928                ready_timeout,
3929                reply: reply_tx,
3930            })
3931            .await
3932            .map_err(|_| SuperviseError::CommandClosed {
3933                module_id: self.inner.module_id.clone(),
3934            })?;
3935        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3936            module_id: self.inner.module_id.clone(),
3937        })?
3938    }
3939
3940    pub async fn reload(&self) -> Result<(), SuperviseError> {
3941        self.reload_requested(None).await
3942    }
3943
3944    pub(crate) async fn reload_requested(
3945        &self,
3946        requester: Option<Principal>,
3947    ) -> Result<(), SuperviseError> {
3948        let (reply_tx, reply_rx) = oneshot::channel();
3949        self.inner
3950            .commands
3951            .send(SupervisorCommand::Reload {
3952                requester,
3953                reply: reply_tx,
3954            })
3955            .await
3956            .map_err(|_| SuperviseError::CommandClosed {
3957                module_id: self.inner.module_id.clone(),
3958            })?;
3959        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3960            module_id: self.inner.module_id.clone(),
3961        })?
3962    }
3963
3964    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3965        self.set_enabled_requested(enabled, None).await
3966    }
3967
3968    pub(crate) async fn set_enabled_requested(
3969        &self,
3970        enabled: bool,
3971        requester: Option<Principal>,
3972    ) -> Result<bool, SuperviseError> {
3973        let (reply_tx, reply_rx) = oneshot::channel();
3974        self.inner
3975            .commands
3976            .send(SupervisorCommand::SetEnabled {
3977                requester,
3978                enabled,
3979                reply: reply_tx,
3980            })
3981            .await
3982            .map_err(|_| SuperviseError::CommandClosed {
3983                module_id: self.inner.module_id.clone(),
3984            })?;
3985        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3986            module_id: self.inner.module_id.clone(),
3987        })?
3988    }
3989
3990    /// The current process's protocol, or the configured protocol when down.
3991    /// A rescan stores the next launch spec without changing how an existing
3992    /// process registers, serves routes, is probed, or exits.
3993    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
3994        let configured = self
3995            .inner
3996            .configuration
3997            .lock()
3998            .map_err(|_| SuperviseError::StatePoisoned {
3999                module_id: Some(self.inner.module_id.clone()),
4000            })?
4001            .spec
4002            .protocol;
4003        let state = lock_snapshot(&self.inner.snapshot)?;
4004        Ok(state.spawned_protocol.unwrap_or(configured))
4005    }
4006
4007    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4008        let configuration =
4009            self.inner
4010                .configuration
4011                .lock()
4012                .map_err(|_| SuperviseError::StatePoisoned {
4013                    module_id: Some(self.inner.module_id.clone()),
4014                })?;
4015        Ok((configuration.spec.clone(), configuration.health.clone()))
4016    }
4017
4018    /// Replace this module's launch spec, keeping its health and drain policy,
4019    /// the way a rescan does for a changed config entry. The running process is
4020    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
4021    #[cfg(any(test, feature = "test-support"))]
4022    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4023        let (_, health) = self.configuration()?;
4024        let drain_timeout_ms = u64::try_from(
4025            self.inner
4026                .effective_drain_timeout
4027                .lock()
4028                .unwrap_or_else(|poisoned| poisoned.into_inner())
4029                .as_millis(),
4030        )
4031        .ok();
4032        self.update_configuration(spec, health, drain_timeout_ms)
4033            .await
4034    }
4035
4036    pub(crate) async fn update_configuration(
4037        &self,
4038        spec: ModuleSpec,
4039        health: HealthConfig,
4040        drain_timeout_ms: Option<u64>,
4041    ) -> Result<(), SuperviseError> {
4042        if spec.module_id != self.inner.module_id {
4043            return Err(SuperviseError::InvalidSpec {
4044                reason: "a supervised module's module_id cannot be changed".to_string(),
4045            });
4046        }
4047        validate_spec(&spec)?;
4048        let (reply_tx, reply_rx) = oneshot::channel();
4049        self.inner
4050            .commands
4051            .send(SupervisorCommand::UpdateConfiguration {
4052                spec: spec.clone(),
4053                health: health.clone(),
4054                drain_timeout_ms,
4055                reply: reply_tx,
4056            })
4057            .await
4058            .map_err(|_| SuperviseError::CommandClosed {
4059                module_id: self.inner.module_id.clone(),
4060            })?;
4061        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4062            module_id: self.inner.module_id.clone(),
4063        })?;
4064        let mut configuration =
4065            self.inner
4066                .configuration
4067                .lock()
4068                .map_err(|_| SuperviseError::StatePoisoned {
4069                    module_id: Some(self.inner.module_id.clone()),
4070                })?;
4071        configuration.spec = spec;
4072        configuration.health = health;
4073        Ok(())
4074    }
4075}
4076
4077impl Drop for SupervisedModuleInner {
4078    fn drop(&mut self) {
4079        let Ok(mut monitor) = self.monitor.lock() else {
4080            return;
4081        };
4082        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4083            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4084                state.state = ModuleState::Stopped;
4085                clear_current_process_facts(state);
4086            });
4087            monitor.abort();
4088        }
4089        let _ = monitor.take();
4090    }
4091}
4092
4093#[derive(Debug)]
4094enum SupervisorCommand {
4095    Drain {
4096        requester: Option<Principal>,
4097        reply: oneshot::Sender<Result<(), SuperviseError>>,
4098    },
4099    Retire {
4100        requester: Option<Principal>,
4101        reply: oneshot::Sender<Result<(), SuperviseError>>,
4102    },
4103    Restart {
4104        requester: Option<Principal>,
4105        /// Operator override for this one restart's drain budget, in ms. `None`
4106        /// uses the module's configured/default budget; `Some(0)` cuts
4107        /// immediately (wedge bounce: a stuck request never settles, so
4108        /// waiting only delays recovery).
4109        drain_timeout_ms: Option<u64>,
4110        /// The module's `spawn_generation` when the request was received, before
4111        /// it waited in the command queue. A queued restart whose module has
4112        /// since spawned a newer process is already satisfied (see the handler).
4113        received_at_generation: u64,
4114        /// When the request entered the command queue, so the handler can log
4115        /// how long it waited behind the loop's other work.
4116        queued_at: Instant,
4117        reply: oneshot::Sender<Result<(), SuperviseError>>,
4118    },
4119    Reload {
4120        requester: Option<Principal>,
4121        reply: oneshot::Sender<Result<(), SuperviseError>>,
4122    },
4123    SetEnabled {
4124        requester: Option<Principal>,
4125        enabled: bool,
4126        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4127    },
4128    UpdateConfiguration {
4129        spec: ModuleSpec,
4130        health: HealthConfig,
4131        /// Per-module drain override from the new config; `None` re-resolves to
4132        /// the supervisor-wide default.
4133        drain_timeout_ms: Option<u64>,
4134        reply: oneshot::Sender<()>,
4135    },
4136    Swap {
4137        requester: Option<Principal>,
4138        /// How long the candidate may take to register and declare itself
4139        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4140        ready_timeout: Option<Duration>,
4141        /// Answered at cutover or failure; the incumbent's drain follows.
4142        reply: oneshot::Sender<Result<(), SuperviseError>>,
4143    },
4144}
4145
4146#[derive(Debug)]
4147pub enum SuperviseError {
4148    InvalidSpec {
4149        reason: String,
4150    },
4151    Spawn {
4152        program: PathBuf,
4153        source: io::Error,
4154        cgroup_path: Option<PathBuf>,
4155    },
4156    Cgroup {
4157        module_id: String,
4158        source: io::Error,
4159    },
4160    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4161    /// than spawn a reserved module without its identity binding.
4162    LaunchNonce {
4163        reason: String,
4164    },
4165    Wait {
4166        module_id: String,
4167        source: io::Error,
4168    },
4169    Kill {
4170        module_id: String,
4171        source: io::Error,
4172    },
4173    Forwarding(ForwardingError),
4174    Registry(RegistryError),
4175    ReloadUnavailable {
4176        module_id: String,
4177        reason: String,
4178    },
4179    /// An operator restart/reload was requested for a module that is currently
4180    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4181    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4182    /// by a restart, so these commands are rejected instead of re-enabling it.
4183    Disabled {
4184        module_id: String,
4185    },
4186    ReloadFailed {
4187        module_id: String,
4188        reason: String,
4189    },
4190    RegistrationStillActive {
4191        module_id: String,
4192        waited: Duration,
4193    },
4194    StatePoisoned {
4195        module_id: Option<String>,
4196    },
4197    CommandClosed {
4198        module_id: String,
4199    },
4200    /// A restart or reload arrived while a swap's candidate was warming. The
4201    /// swap owns the module until it cuts over or fails; a stop or disable
4202    /// would have aborted it instead.
4203    SwapInProgress {
4204        module_id: String,
4205    },
4206    /// A swap was refused before anything was spawned.
4207    SwapRefused {
4208        module_id: String,
4209        reason: SwapRefusal,
4210    },
4211    /// A swap spawned a candidate and gave up on it. The candidate has been
4212    /// killed and its slot freed; the incumbent was left serving and was never
4213    /// drained, except in the one `CutoverLost` case described on that arm.
4214    SwapFailed {
4215        module_id: String,
4216        arm: SwapFailureArm,
4217        detail: String,
4218        /// How the candidate exited, when it exited on its own before the
4219        /// supervisor gave up on it.
4220        candidate_exit: Option<ExitReport>,
4221    },
4222}
4223
4224/// Why a swap was refused before a candidate was spawned.
4225#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4226pub enum SwapRefusal {
4227    /// The module's config does not declare `overlap: "safe"`.
4228    OverlapExclusive,
4229    /// The module is not registered, so there is no incumbent to keep serving
4230    /// and nothing a swap would improve on; a plain restart is the tool.
4231    NotRegistered,
4232    /// The module does not speak the subc wire, so a candidate could never
4233    /// register or declare itself ready.
4234    ProtocolNone,
4235    /// The supervisor lacks the forwarding table (to cut routes over) or the
4236    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4237    NotConfigured,
4238    /// A swap is already open for this module.
4239    AlreadySwapping,
4240}
4241
4242impl SwapRefusal {
4243    pub fn as_str(self) -> &'static str {
4244        match self {
4245            Self::OverlapExclusive => "overlap_exclusive",
4246            Self::NotRegistered => "not_registered",
4247            Self::ProtocolNone => "protocol_none",
4248            Self::NotConfigured => "not_configured",
4249            Self::AlreadySwapping => "already_swapping",
4250        }
4251    }
4252}
4253
4254/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4255/// serving and undrained; see `CutoverLost`.
4256#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4257pub enum SwapFailureArm {
4258    /// The candidate process could not be started.
4259    SpawnFailed,
4260    /// The candidate did not register within the readiness budget.
4261    NeverRegistered,
4262    /// The candidate registered but did not declare itself ready in time.
4263    NeverReady,
4264    /// The candidate exited before cutover.
4265    CandidateExited,
4266    /// The candidate declared itself ready but failed its health probe.
4267    CandidateUnhealthy,
4268    /// An operator stop, disable or retire arrived while the candidate warmed.
4269    /// The candidate was killed and the operator's command then carried out on
4270    /// the incumbent.
4271    Interrupted,
4272    /// The candidate's connection closed at the moment of cutover. If it
4273    /// closed before forwarding moved, the incumbent is untouched. If it closed
4274    /// between the forwarding and registry halves of cutover, forwarding can no
4275    /// longer route to the incumbent, so the module is restarted plainly.
4276    CutoverLost,
4277}
4278
4279impl SwapFailureArm {
4280    pub fn as_str(self) -> &'static str {
4281        match self {
4282            Self::SpawnFailed => "spawn_failed",
4283            Self::NeverRegistered => "never_registered",
4284            Self::NeverReady => "never_ready",
4285            Self::CandidateExited => "candidate_exited",
4286            Self::CandidateUnhealthy => "candidate_unhealthy",
4287            Self::Interrupted => "interrupted",
4288            Self::CutoverLost => "cutover_lost",
4289        }
4290    }
4291}
4292
4293impl fmt::Display for SuperviseError {
4294    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4295        match self {
4296            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4297            Self::Spawn {
4298                program,
4299                source,
4300                cgroup_path: Some(cgroup_path),
4301            } => write!(
4302                f,
4303                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4304                cgroup_path.display(),
4305                program.display()
4306            ),
4307            Self::Spawn {
4308                program,
4309                source,
4310                cgroup_path: None,
4311            } => write!(
4312                f,
4313                "failed to spawn module '{}': {source}",
4314                program.display()
4315            ),
4316            Self::Cgroup { module_id, source } => {
4317                write!(
4318                    f,
4319                    "failed to prepare cgroup for module '{module_id}': {source}"
4320                )
4321            }
4322            Self::LaunchNonce { reason } => {
4323                write!(
4324                    f,
4325                    "failed to generate reserved-module launch nonce: {reason}"
4326                )
4327            }
4328            Self::Wait { module_id, source } => {
4329                write!(f, "failed to wait for module '{module_id}': {source}")
4330            }
4331            Self::Kill { module_id, source } => {
4332                write!(f, "failed to kill module '{module_id}': {source}")
4333            }
4334            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4335            Self::Registry(err) => write!(f, "registry error: {err}"),
4336            Self::ReloadUnavailable { module_id, reason } => {
4337                write!(f, "reload unavailable for module '{module_id}': {reason}")
4338            }
4339            Self::Disabled { module_id } => {
4340                write!(
4341                    f,
4342                    "module '{module_id}' is disabled; enable it before restart or reload"
4343                )
4344            }
4345            Self::ReloadFailed { module_id, reason } => {
4346                write!(f, "reload failed for module '{module_id}': {reason}")
4347            }
4348            Self::RegistrationStillActive { module_id, waited } => write!(
4349                f,
4350                "module '{module_id}' registration remained active after waiting {waited:?}"
4351            ),
4352            Self::StatePoisoned { module_id } => match module_id {
4353                Some(module_id) => {
4354                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4355                }
4356                None => write!(f, "supervisor state was poisoned"),
4357            },
4358            Self::CommandClosed { module_id } => {
4359                write!(
4360                    f,
4361                    "supervisor command channel for module '{module_id}' is closed"
4362                )
4363            }
4364            Self::SwapInProgress { module_id } => write!(
4365                f,
4366                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4367            ),
4368            Self::SwapRefused { module_id, reason } => match reason {
4369                SwapRefusal::OverlapExclusive => write!(
4370                    f,
4371                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4372                ),
4373                SwapRefusal::NotRegistered => write!(
4374                    f,
4375                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4376                ),
4377                SwapRefusal::ProtocolNone => write!(
4378                    f,
4379                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4380                ),
4381                SwapRefusal::NotConfigured => write!(
4382                    f,
4383                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4384                ),
4385                SwapRefusal::AlreadySwapping => {
4386                    write!(f, "module '{module_id}' is already being swapped")
4387                }
4388            },
4389            Self::SwapFailed {
4390                module_id,
4391                arm,
4392                detail,
4393                ..
4394            } => write!(
4395                f,
4396                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4397                arm.as_str()
4398            ),
4399        }
4400    }
4401}
4402
4403impl Error for SuperviseError {
4404    fn source(&self) -> Option<&(dyn Error + 'static)> {
4405        match self {
4406            Self::Spawn { source, .. }
4407            | Self::Cgroup { source, .. }
4408            | Self::Wait { source, .. }
4409            | Self::Kill { source, .. } => Some(source),
4410            Self::Forwarding(err) => Some(err),
4411            Self::Registry(err) => Some(err),
4412            Self::LaunchNonce { .. }
4413            | Self::InvalidSpec { .. }
4414            | Self::ReloadUnavailable { .. }
4415            | Self::Disabled { .. }
4416            | Self::ReloadFailed { .. }
4417            | Self::RegistrationStillActive { .. }
4418            | Self::StatePoisoned { .. }
4419            | Self::CommandClosed { .. }
4420            | Self::SwapInProgress { .. }
4421            | Self::SwapRefused { .. }
4422            | Self::SwapFailed { .. } => None,
4423        }
4424    }
4425}
4426
4427pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4428    if spec.module_id.trim().is_empty() {
4429        return Err(SuperviseError::InvalidSpec {
4430            reason: "module_id must not be empty".to_string(),
4431        });
4432    }
4433
4434    Ok(())
4435}
4436
4437#[derive(Debug, Default)]
4438struct HealthProbeRuntime {
4439    configured_health: Option<HealthConfig>,
4440    registered_connection: Option<crate::ConnectionId>,
4441    advertised: bool,
4442    next_probe_at: Option<Instant>,
4443    probe_index: u64,
4444}
4445
4446fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4447    lock_snapshot(snapshot)
4448        .ok()
4449        .and_then(|state| state.spawned_protocol)
4450        .unwrap_or(spec.protocol)
4451}
4452
4453impl HealthProbeRuntime {
4454    fn refresh_registration(
4455        &mut self,
4456        spec: &ModuleSpec,
4457        runtime: &SupervisorRuntimeConfig,
4458        registry: &Registry,
4459        snapshot: &SharedSnapshot,
4460    ) {
4461        if self.configured_health.as_ref() != Some(&runtime.health) {
4462            self.configured_health = Some(runtime.health.clone());
4463            self.next_probe_at = None;
4464            self.registered_connection = None;
4465            self.probe_index = 0;
4466        }
4467        // A non-wire process never registers. Only an explicitly configured
4468        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4469        // health failure for that kind of process.
4470        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4471            self.registered_connection = None;
4472            self.advertised = runtime.health.http.is_some();
4473            if !self.advertised {
4474                self.next_probe_at = None;
4475                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4476                    let unknown = ModuleHealthStatus::default();
4477                    if state.health != unknown {
4478                        state.health = unknown;
4479                    }
4480                });
4481            } else if self.next_probe_at.is_none() {
4482                self.next_probe_at = Some(
4483                    Instant::now()
4484                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4485                );
4486            }
4487            return;
4488        }
4489
4490        let registration = match registry.get_module(&spec.module_id) {
4491            Ok(registration) => registration,
4492            Err(err) => {
4493                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4494                self.advertised = false;
4495                self.next_probe_at = None;
4496                return;
4497            }
4498        };
4499
4500        let Some(registration) = registration else {
4501            self.registered_connection = None;
4502            self.advertised = false;
4503            self.next_probe_at = None;
4504            return;
4505        };
4506
4507        let advertised = registration
4508            .control_ops
4509            .iter()
4510            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4511        if !advertised {
4512            self.registered_connection = Some(registration.connection_id);
4513            self.advertised = false;
4514            self.next_probe_at = None;
4515            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4516                if state.health.status != SupervisorHealthStatus::Unknown
4517                    || state.health.consecutive_failures != 0
4518                    || state.health.last_probe_ms.is_some()
4519                    || state.health.detail.is_some()
4520                    || state.health.metrics.is_some()
4521                {
4522                    state.health.status = SupervisorHealthStatus::Unknown;
4523                    state.health.consecutive_failures = 0;
4524                    state.health.last_probe_ms = None;
4525                    state.health.detail = None;
4526                    state.health.metrics = None;
4527                }
4528            });
4529            return;
4530        }
4531
4532        let reregistered = self.registered_connection != Some(registration.connection_id);
4533        self.registered_connection = Some(registration.connection_id);
4534        self.advertised = true;
4535        if reregistered || self.next_probe_at.is_none() {
4536            self.probe_index = 0;
4537            self.next_probe_at = Some(
4538                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4539            );
4540            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4541                state.health.status = SupervisorHealthStatus::Unknown;
4542                state.health.consecutive_failures = 0;
4543                state.health.detail = None;
4544                state.health.metrics = None;
4545            });
4546        }
4547    }
4548
4549    fn wake_after(&self) -> Option<Duration> {
4550        if !self.advertised {
4551            return None;
4552        }
4553        self.next_probe_at
4554            .map(|next| next.saturating_duration_since(Instant::now()))
4555    }
4556
4557    fn due(&self) -> bool {
4558        self.advertised
4559            && self
4560                .next_probe_at
4561                .is_some_and(|next| Instant::now() >= next)
4562    }
4563
4564    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4565        self.probe_index = self.probe_index.wrapping_add(1);
4566        self.next_probe_at = Some(
4567            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4568        );
4569    }
4570}
4571
4572/// What a failed health probe actually OBSERVED, kept apart from how it reads.
4573///
4574/// This was a struct with a single `message: String`, and every one of the
4575/// fifteen construction sites collapsed into it. Each site knows exactly what it
4576/// saw -- the lane is gone, the module did not answer in time, the module
4577/// answered with the wrong thing -- and `handle_health_probe_failure` then
4578/// treated all of them identically: increment a counter, compare to a threshold,
4579/// restart the module. THE DISTINCTION EXISTED AT EVERY CALL SITE AND WAS
4580/// DESTROYED BEFORE THE DECISION THAT NEEDED IT.
4581///
4582/// The distinction that matters is not severity, it is EVIDENTIAL WEIGHT:
4583///
4584/// * `LaneDead` is PROOF. The module's control connection is gone; nothing will
4585///   answer on it again.
4586/// * `NoAnswer` is ABSENCE OF EVIDENCE. It is consistent with a wedged module
4587///   AND with a perfectly healthy one that lost a CPU race -- which is what
4588///   happens under machine load, and is how this supervisor killed a healthy
4589///   module three times in one day.
4590/// * `BadAnswer` proves the module is ALIVE. It replied; the reply was wrong.
4591///   Restarting on it is defensible, but it is not the silence case and should
4592///   never be counted as one.
4593/// * `Misconfigured` is a daemon-side fault. The module has not been asked
4594///   anything, so it cannot be evidence about the module at all.
4595///
4596/// The asymmetry is the whole point: under saturation the WEAKEST signal is the
4597/// one that fires most often, and while every variant collapsed into one string
4598/// it carried the same weight as the strongest.
4599///
4600/// LIVE BEHAVIOUR TODAY, stated here because this doc block describes the
4601/// DESIGN and a reader stopping at it gets the build backwards: the restart
4602/// decision does NOT yet consult this classification -- consecutive `NoAnswer`
4603/// probes still increment the failure streak and drive escalation at the
4604/// threshold (see `is_proof_of_death` below for why that is deliberate and
4605/// what gates the change). Absence of evidence restarts modules today.
4606#[derive(Debug)]
4607enum HealthProbeEvidence {
4608    /// The module's control lane is gone. Proof of death.
4609    LaneDead,
4610    /// No reply within the deadline. Proves nothing about the module's state.
4611    NoAnswer,
4612    /// The module replied, but not with a usable health report. Proves it is alive.
4613    BadAnswer,
4614    /// The daemon could not ask. Says nothing about the module.
4615    Misconfigured,
4616}
4617
4618#[derive(Debug)]
4619struct HealthProbeError {
4620    evidence: HealthProbeEvidence,
4621    message: String,
4622}
4623
4624impl HealthProbeError {
4625    fn lane_dead(message: impl Into<String>) -> Self {
4626        Self::with(HealthProbeEvidence::LaneDead, message)
4627    }
4628
4629    fn no_answer(message: impl Into<String>) -> Self {
4630        Self::with(HealthProbeEvidence::NoAnswer, message)
4631    }
4632
4633    fn bad_answer(message: impl Into<String>) -> Self {
4634        Self::with(HealthProbeEvidence::BadAnswer, message)
4635    }
4636
4637    fn misconfigured(message: impl Into<String>) -> Self {
4638        Self::with(HealthProbeEvidence::Misconfigured, message)
4639    }
4640
4641    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4642        Self {
4643            evidence,
4644            message: message.into(),
4645        }
4646    }
4647
4648    /// Whether this observation is proof the module cannot serve.
4649    ///
4650    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4651    /// variant that fires under CPU starvation, and treating it as proof is the
4652    /// defect this enum exists to make impossible to reintroduce silently.
4653    ///
4654    /// NOT YET CONSULTED BY THE RESTART DECISION, deliberately. Requiring proof
4655    /// to restart also needs a bound for the case it excludes -- a genuinely
4656    /// wedged module, alive but never answering -- and that bound must come from
4657    /// the distribution of real late-answer latencies, which nothing measures
4658    /// yet. Landing the classification first makes the later change a one-line
4659    /// decision against evidence that already exists, rather than two unproven
4660    /// changes at once.
4661    #[allow(dead_code)]
4662    fn is_proof_of_death(&self) -> bool {
4663        matches!(self.evidence, HealthProbeEvidence::LaneDead)
4664    }
4665
4666    /// Short stable label for logs and the health snapshot.
4667    ///
4668    /// An operator reading `ck health` currently cannot tell "the module is gone"
4669    /// from "the module did not answer in five seconds", because both render as
4670    /// prose in the same field. These labels are what make the two
4671    /// distinguishable at a glance, and they are what a later restart-policy
4672    /// change will be argued from.
4673    fn label(&self) -> &'static str {
4674        match self.evidence {
4675            HealthProbeEvidence::LaneDead => "lane-dead",
4676            HealthProbeEvidence::NoAnswer => "no-answer",
4677            HealthProbeEvidence::BadAnswer => "bad-answer",
4678            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4679        }
4680    }
4681}
4682
4683impl fmt::Display for HealthProbeError {
4684    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4685        f.write_str(&self.message)
4686    }
4687}
4688
4689async fn run_health_probe_cycle(
4690    spec: &ModuleSpec,
4691    runtime: &SupervisorRuntimeConfig,
4692    registry: &Registry,
4693    process_liveness: &SupervisorProcessLiveness,
4694    snapshot: &SharedSnapshot,
4695    child: &mut Option<SupervisedChild>,
4696) {
4697    let now_ms = unix_ms_now();
4698    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4699        .then_some(runtime.health.http.as_deref())
4700        .flatten();
4701    let result = match http {
4702        Some(url) => probe_http_health(url, runtime.health.deadline).await,
4703        None => probe_module_health(&spec.module_id, runtime, None).await,
4704    };
4705    match result {
4706        Ok(report) => {
4707            handle_health_report(
4708                spec,
4709                runtime,
4710                registry,
4711                process_liveness,
4712                snapshot,
4713                child,
4714                report,
4715                now_ms,
4716            )
4717            .await;
4718        }
4719        Err(err) => {
4720            if http.is_some() {
4721                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4722                    state.health.status = SupervisorHealthStatus::Failing;
4723                });
4724            }
4725            handle_health_probe_failure(
4726                spec,
4727                runtime,
4728                registry,
4729                process_liveness,
4730                snapshot,
4731                child,
4732                err,
4733                now_ms,
4734            )
4735            .await;
4736        }
4737    }
4738}
4739
4740pub(crate) struct HttpProbeTarget<'a> {
4741    address: std::net::SocketAddr,
4742    localhost: bool,
4743    authority: &'a str,
4744    path: String,
4745}
4746
4747/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
4748/// or TLS. A URL cannot turn a local health check into an outbound connection.
4749pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4750    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4751        return Err("must not contain whitespace, controls, or a fragment".into());
4752    }
4753    let rest = url
4754        .strip_prefix("http://")
4755        .ok_or("must use plain http://")?;
4756    let split = rest.find(['/', '?']).unwrap_or(rest.len());
4757    let (authority, suffix) = rest.split_at(split);
4758    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4759        ("::1", rest)
4760    } else {
4761        let split = authority.find(':').unwrap_or(authority.len());
4762        authority.split_at(split)
4763    };
4764    let ip: std::net::IpAddr = match host {
4765        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4766        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4767        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4768    };
4769    let port = if port.is_empty() {
4770        80
4771    } else {
4772        port.strip_prefix(':')
4773            .and_then(|p| p.parse::<u16>().ok())
4774            .filter(|p| *p > 0)
4775            .ok_or("must have a valid nonzero TCP port")?
4776    };
4777    let path = if suffix.is_empty() {
4778        "/".into()
4779    } else if suffix.starts_with('?') {
4780        format!("/{suffix}")
4781    } else {
4782        suffix.into()
4783    };
4784    Ok(HttpProbeTarget {
4785        address: std::net::SocketAddr::new(ip, port),
4786        localhost: host == "localhost",
4787        authority,
4788        path,
4789    })
4790}
4791
4792async fn probe_http_health(
4793    url: &str,
4794    deadline: Duration,
4795) -> Result<HealthReport, HealthProbeError> {
4796    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4797    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4798    // Keep partial diagnostics outside the timed future so cancellation does
4799    // not discard a status line or body bytes already received.
4800    let mut response_status = String::new();
4801    let mut body = Vec::new();
4802    let probe = async {
4803        // Resolve localhost ourselves so a hosts-file override cannot turn
4804        // this into an outbound request, while IPv6-only local servers work.
4805        let connection = match tokio::net::TcpStream::connect(target.address).await {
4806            Err(_) if target.localhost => {
4807                tokio::net::TcpStream::connect((
4808                    std::net::Ipv6Addr::LOCALHOST,
4809                    target.address.port(),
4810                ))
4811                .await
4812            }
4813            result => result,
4814        };
4815        let mut stream = connection.map_err(|error| {
4816            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4817        })?;
4818        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4819            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4820        let mut reader = BufReader::new(stream);
4821        let mut budget = 16 * 1024;
4822        let status = http_line(&mut reader, &mut budget).await?;
4823        let mut words = status.split_ascii_whitespace();
4824        let version = words.next();
4825        let code = words
4826            .next()
4827            .filter(|word| word.len() == 3)
4828            .and_then(|word| word.parse::<u16>().ok());
4829        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4830            || !code.is_some_and(|code| (100..600).contains(&code))
4831        {
4832            return Err(HealthProbeError::bad_answer(format!(
4833                "invalid HTTP status: {status}"
4834            )));
4835        }
4836        let code = code.expect("validated status code");
4837        response_status = status.clone();
4838        let mut length = None;
4839        let mut chunked = false;
4840        loop {
4841            let line = http_line(&mut reader, &mut budget).await?;
4842            if line.is_empty() {
4843                break;
4844            }
4845            if let Some((name, value)) = line.split_once(':') {
4846                if name.eq_ignore_ascii_case("content-length") {
4847                    length = Some(value.trim().parse::<u64>().map_err(|_| {
4848                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
4849                    })?);
4850                } else if name.eq_ignore_ascii_case("transfer-encoding") {
4851                    chunked = value.trim().eq_ignore_ascii_case("chunked");
4852                }
4853            }
4854        }
4855        if chunked {
4856            while body.len() < 200 {
4857                let line = http_line(&mut reader, &mut budget).await?;
4858                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4859                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4860                if size == 0 {
4861                    break;
4862                }
4863                let count = size.min((200 - body.len()) as u64) as usize;
4864                let start = body.len();
4865                (&mut reader)
4866                    .take(count as u64)
4867                    .read_to_end(&mut body)
4868                    .await
4869                    .map_err(|error| {
4870                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4871                    })?;
4872                if body.len() - start != count {
4873                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4874                }
4875                if size > count as u64 || body.len() == 200 {
4876                    break;
4877                }
4878                if !http_line(&mut reader, &mut budget).await?.is_empty() {
4879                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4880                }
4881            }
4882        } else {
4883            reader
4884                .take(length.unwrap_or(200).min(200))
4885                .read_to_end(&mut body)
4886                .await
4887                .map_err(|error| {
4888                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4889                })?;
4890        }
4891        if (200..300).contains(&code) {
4892            Ok(HealthReport::ok())
4893        } else {
4894            Err(HealthProbeError::bad_answer(
4895                "HTTP health endpoint returned non-2xx",
4896            ))
4897        }
4898    };
4899    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4900        Err(HealthProbeError::no_answer(format!(
4901            "HTTP probe timed out after {deadline:?}"
4902        )))
4903    });
4904    if let Err(error) = &mut result {
4905        if !response_status.is_empty() {
4906            error.message = format!(
4907                "{}; {response_status}: {}",
4908                error.message,
4909                String::from_utf8_lossy(&body)
4910            );
4911        }
4912    }
4913    result
4914}
4915
4916async fn http_line(
4917    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4918    remaining: &mut usize,
4919) -> Result<String, HealthProbeError> {
4920    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4921    let mut line = Vec::new();
4922    (&mut *reader)
4923        .take(*remaining as u64)
4924        .read_until(b'\n', &mut line)
4925        .await
4926        .map_err(|error| {
4927            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4928        })?;
4929    *remaining -= line.len();
4930    if !line.ends_with(b"\r\n") {
4931        return Err(HealthProbeError::bad_answer(
4932            "HTTP headers are incomplete or exceed 16 KiB",
4933        ));
4934    }
4935    line.truncate(line.len() - 2);
4936    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4937}
4938
4939async fn probe_module_health(
4940    module_id: &str,
4941    runtime: &SupervisorRuntimeConfig,
4942    drain_deadline: Option<Instant>,
4943) -> Result<HealthReport, HealthProbeError> {
4944    let Some(forwarding) = runtime.forwarding.as_ref() else {
4945        return Err(HealthProbeError::misconfigured(
4946            "supervisor was not configured with a forwarding table",
4947        ));
4948    };
4949    let probe_started_at = Instant::now();
4950    let mut deadline = probe_started_at + runtime.health.deadline;
4951    if let Some(drain_deadline) = drain_deadline {
4952        deadline = deadline.min(drain_deadline);
4953    }
4954    let pending = if drain_deadline.is_some() {
4955        forwarding.begin_drain_health_probe_rpc_for(
4956            module_id,
4957            MODULE_CONTROL_OP_HEALTH_CHECK,
4958            probe_started_at,
4959            deadline,
4960        )
4961    } else {
4962        forwarding.begin_health_probe_rpc_for(
4963            module_id,
4964            MODULE_CONTROL_OP_HEALTH_CHECK,
4965            probe_started_at,
4966            deadline,
4967        )
4968    }
4969    .map_err(|err| {
4970        // The endpoint is not registered, so there is no live control lane to
4971        // ask. That is the module being absent, not slow.
4972        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4973    })?;
4974    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4975}
4976
4977/// [`probe_module_health`] for one endpoint rather than the id's active one.
4978///
4979/// A swap probes two processes that no by-id lookup reaches: its candidate
4980/// before cutover, and its superseded incumbent (for busy gauges) while the
4981/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
4982/// bounds the by-id drain probe.
4983async fn probe_endpoint_health(
4984    endpoint: crate::ModuleEndpointId,
4985    runtime: &SupervisorRuntimeConfig,
4986    deadline_cap: Option<Instant>,
4987) -> Result<HealthReport, HealthProbeError> {
4988    let Some(forwarding) = runtime.forwarding.as_ref() else {
4989        return Err(HealthProbeError::misconfigured(
4990            "supervisor was not configured with a forwarding table",
4991        ));
4992    };
4993    let probe_started_at = Instant::now();
4994    let mut deadline = probe_started_at + runtime.health.deadline;
4995    if let Some(cap) = deadline_cap {
4996        deadline = deadline.min(cap);
4997    }
4998    let pending = forwarding
4999        .begin_endpoint_health_probe_rpc_for(
5000            endpoint,
5001            MODULE_CONTROL_OP_HEALTH_CHECK,
5002            probe_started_at,
5003            deadline,
5004        )
5005        .map_err(|err| {
5006            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5007        })?;
5008    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5009}
5010
5011/// Send a begun health probe and classify its answer.
5012async fn await_health_probe(
5013    forwarding: &ForwardingTable,
5014    pending: PendingModuleControlRpc,
5015    deadline: Instant,
5016    probe_budget: Duration,
5017) -> Result<HealthReport, HealthProbeError> {
5018    let PendingModuleControlRpc {
5019        endpoint,
5020        module_sink,
5021        negotiated_ver,
5022        corr,
5023        receiver,
5024    } = pending;
5025    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5026        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5027    })?;
5028    let frame = Frame::build_with_version(
5029        negotiated_ver,
5030        FrameType::Request,
5031        control_flags(),
5032        0,
5033        0,
5034        corr,
5035        body,
5036    )
5037    .map_err(|err| {
5038        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5039    })?;
5040
5041    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
5042    // blocks waiting for capacity when the module's egress queue is full, and an
5043    // unbounded await here freezes the whole supervision actor (it stops polling
5044    // Child::wait and supervisor commands), making the module unrecoverable
5045    // in-band. On timeout the probe fails like any transport failure.
5046    match timeout_at(deadline, module_sink.send(frame)).await {
5047        Ok(Ok(())) => {}
5048        Ok(Err(err)) => {
5049            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5050            // A closed sink means the module's egress channel is gone -- the
5051            // receiving half is dropped when its connection tears down. Proof.
5052            return Err(HealthProbeError::lane_dead(format!(
5053                "failed to send health.check: {err}"
5054            )));
5055        }
5056        Err(_elapsed) => {
5057            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5058            // A full egress queue means the module is not draining its socket, which
5059            // is consistent with a wedged module AND with one whose reader is merely
5060            // starved. Silence, not proof.
5061            return Err(HealthProbeError::no_answer(
5062                "health.check send timed out before enqueue (module egress full)",
5063            ));
5064        }
5065    }
5066
5067    match timeout_at(deadline, receiver).await {
5068        // Each arm records WHAT WAS OBSERVED. Four of them are the module
5069        // demonstrably answering -- rejected, non-health, malformed, wrong op --
5070        // and those prove it is alive even though the probe failed.
5071        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5072            response.health_report().ok_or_else(|| {
5073                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5074            })
5075        }
5076        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5077            format!("health.check rejected: {}", body.message),
5078        )),
5079        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5080            Err(HealthProbeError::lane_dead(message))
5081        }
5082        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5083            Err(HealthProbeError::bad_answer(message))
5084        }
5085        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5086            Err(HealthProbeError::bad_answer(format!(
5087                "expected module-control op '{expected}', got '{actual}'"
5088            )))
5089        }
5090        // A reply that crosses the deadline before this waiter observes it is
5091        // still proof of life. The forwarding path records its end-to-end latency
5092        // before delivering this classification.
5093        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5094            "module answered health.check after its daemon deadline",
5095        )),
5096        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5097            "health.check waiter was canceled before the module responded",
5098        )),
5099        Err(_) => {
5100            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5101            Err(HealthProbeError::no_answer(format!(
5102                "module did not answer health.check within {probe_budget:?}"
5103            )))
5104        }
5105    }
5106}
5107
5108#[allow(clippy::too_many_arguments)]
5109async fn handle_health_report(
5110    spec: &ModuleSpec,
5111    runtime: &SupervisorRuntimeConfig,
5112    registry: &Registry,
5113    process_liveness: &SupervisorProcessLiveness,
5114    snapshot: &SharedSnapshot,
5115    child: &mut Option<SupervisedChild>,
5116    report: HealthReport,
5117    now_ms: u64,
5118) {
5119    let status = supervisor_health_status(report.status);
5120    let detail = report.detail.clone();
5121    let metrics = truncate_health_metrics(report.metrics);
5122    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5123        state.health.status = status;
5124        state.health.last_probe_ms = Some(now_ms);
5125        state.health.detail = detail.clone();
5126        state.health.metrics = metrics.clone();
5127        state.health.consecutive_failures = 0;
5128    });
5129
5130    let action = match report.status {
5131        HealthStatus::Ok => return,
5132        HealthStatus::Degraded => runtime.health.on_degraded,
5133        HealthStatus::Failing => runtime.health.on_failing,
5134    };
5135    apply_l3_health_action(
5136        spec,
5137        runtime,
5138        registry,
5139        process_liveness,
5140        snapshot,
5141        child,
5142        status,
5143        detail.as_deref(),
5144        action,
5145        now_ms,
5146    )
5147    .await;
5148}
5149
5150#[allow(clippy::too_many_arguments)]
5151async fn handle_health_probe_failure(
5152    spec: &ModuleSpec,
5153    runtime: &SupervisorRuntimeConfig,
5154    registry: &Registry,
5155    process_liveness: &SupervisorProcessLiveness,
5156    snapshot: &SharedSnapshot,
5157    child: &mut Option<SupervisedChild>,
5158    err: HealthProbeError,
5159    now_ms: u64,
5160) {
5161    let threshold = runtime.health.failure_threshold.max(1);
5162    let mut failures = 0;
5163    // Carry the evidence class into the operator-visible detail. Without it,
5164    // "module did not answer within 5s" and "the control lane is gone" are two
5165    // prose strings in the same field, and the reader has to know the codebase to
5166    // tell which one is proof of anything.
5167    let detail = format!("[{}] {err}", err.label());
5168    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5169        // A failed wire probe invalidates the last report, even before the
5170        // restart threshold. HTTP probes already mark failures as Failing.
5171        if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5172            state.health.status = SupervisorHealthStatus::Unknown;
5173        }
5174        state.health.last_probe_ms = Some(now_ms);
5175        state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5176        state.health.detail = Some(detail.clone());
5177        state.health.metrics = None;
5178        failures = state.health.consecutive_failures;
5179    });
5180
5181    if failures < threshold {
5182        warn!(
5183            module_id = %spec.module_id,
5184            consecutive_failures = failures,
5185            threshold,
5186            evidence = err.label(),
5187            detail = %detail,
5188            "health.check probe failed"
5189        );
5190        return;
5191    }
5192
5193    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5194        state.state = ModuleState::Unresponsive;
5195        state.health.status = SupervisorHealthStatus::Unresponsive;
5196    });
5197    // The evidence class is logged at the kill site because this is the line an
5198    // operator reads after an unexplained restart. A streak of `no-answer` under
5199    // machine load is the known false-positive shape; a `lane-dead` is not.
5200    if runtime.health.critical {
5201        error!(
5202            module_id = %spec.module_id,
5203            status = "unresponsive",
5204            evidence = err.label(),
5205            detail = %detail,
5206            "critical module health alert"
5207        );
5208    } else {
5209        warn!(
5210            module_id = %spec.module_id,
5211            status = "unresponsive",
5212            evidence = err.label(),
5213            detail = %detail,
5214            "module health threshold breached"
5215        );
5216    }
5217    if let Err(err) = health_restart_child(
5218        spec,
5219        runtime,
5220        registry,
5221        process_liveness,
5222        snapshot,
5223        child,
5224        SupervisorHealthStatus::Unresponsive,
5225        Some(&detail),
5226        now_ms,
5227    )
5228    .await
5229    {
5230        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5231    }
5232}
5233
5234#[allow(clippy::too_many_arguments)]
5235async fn apply_l3_health_action(
5236    spec: &ModuleSpec,
5237    runtime: &SupervisorRuntimeConfig,
5238    registry: &Registry,
5239    process_liveness: &SupervisorProcessLiveness,
5240    snapshot: &SharedSnapshot,
5241    child: &mut Option<SupervisedChild>,
5242    status: SupervisorHealthStatus,
5243    detail: Option<&str>,
5244    action: HealthAction,
5245    now_ms: u64,
5246) {
5247    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5248    match action {
5249        HealthAction::Report => {
5250            info!(
5251                module_id = %spec.module_id,
5252                status = ?status,
5253                detail,
5254                "module reported non-ok health"
5255            );
5256        }
5257        HealthAction::Alert => {
5258            error!(
5259                module_id = %spec.module_id,
5260                status = ?status,
5261                detail,
5262                "module health alert"
5263            );
5264        }
5265        HealthAction::Restart => {
5266            if let Err(err) = health_restart_child(
5267                spec,
5268                runtime,
5269                registry,
5270                process_liveness,
5271                snapshot,
5272                child,
5273                status,
5274                detail,
5275                now_ms,
5276            )
5277            .await
5278            {
5279                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5280            }
5281        }
5282    }
5283}
5284
5285#[allow(clippy::too_many_arguments)]
5286async fn health_restart_child(
5287    spec: &ModuleSpec,
5288    runtime: &SupervisorRuntimeConfig,
5289    registry: &Registry,
5290    process_liveness: &SupervisorProcessLiveness,
5291    snapshot: &SharedSnapshot,
5292    child: &mut Option<SupervisedChild>,
5293    status: SupervisorHealthStatus,
5294    detail: Option<&str>,
5295    now_ms: u64,
5296) -> Result<(), SuperviseError> {
5297    let (enabled, schedule) = {
5298        let mut state = lock_snapshot(snapshot)?;
5299        let enabled = state.enabled;
5300        let schedule = if enabled {
5301            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5302        } else {
5303            None
5304        };
5305        (enabled, schedule)
5306    };
5307
5308    if !enabled {
5309        return Err(SuperviseError::Disabled {
5310            module_id: spec.module_id.clone(),
5311        });
5312    }
5313
5314    if schedule.is_none() {
5315        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5316        error!(
5317            module_id = %spec.module_id,
5318            status = ?status,
5319            detail,
5320            max_restarts = runtime.restart_policy.max_restarts,
5321            window_secs = runtime.restart_policy.window.as_secs(),
5322            reason = %runtime.restart_policy.budget_exhausted_detail(),
5323            "health restart budget exhausted; marking module failed"
5324        );
5325        let stop_notice = begin_forwarding_drain_if_configured(
5326            spec,
5327            runtime,
5328            registry,
5329            snapshot,
5330            Some(true),
5331            RouteCloseReason::Disable,
5332        )
5333        .await?;
5334        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5335            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5336        })?;
5337        drain_optional_child(
5338            &spec.module_id,
5339            spec.protocol,
5340            stop_notice,
5341            registry,
5342            runtime.forwarding.as_deref(),
5343            snapshot,
5344            &runtime.terminal_ring,
5345            &runtime.spawn_events,
5346            child,
5347            runtime.drain_timeout,
5348            ModuleState::Failed,
5349            Some(true),
5350        )
5351        .await?;
5352        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5353        return Ok(());
5354    }
5355
5356    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5357    let mut restart_count = 0;
5358    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5359        restart_count = state.crash_restarts.len();
5360        state.state = ModuleState::Unresponsive;
5361        state.health.status = status;
5362        state.health.last_action = Some(HealthAction::Restart.to_string());
5363        state.health.last_action_ms = Some(now_ms);
5364    })?;
5365    warn!(
5366        module_id = %spec.module_id,
5367        status = ?status,
5368        detail,
5369        restart_count,
5370        restart_in_window = schedule.restart_in_window,
5371        delay_ms = schedule.delay.as_millis() as u64,
5372        "health-triggered module restart"
5373    );
5374
5375    let stop_notice = begin_forwarding_drain_if_configured(
5376        spec,
5377        runtime,
5378        registry,
5379        snapshot,
5380        Some(true),
5381        RouteCloseReason::Restart,
5382    )
5383    .await?;
5384    drain_optional_child(
5385        &spec.module_id,
5386        spec.protocol,
5387        stop_notice,
5388        registry,
5389        runtime.forwarding.as_deref(),
5390        snapshot,
5391        &runtime.terminal_ring,
5392        &runtime.spawn_events,
5393        child,
5394        runtime.drain_timeout,
5395        ModuleState::Restarting,
5396        Some(true),
5397    )
5398    .await?;
5399    schedule_respawn(
5400        runtime,
5401        snapshot,
5402        &spec.module_id,
5403        schedule.delay,
5404        RespawnKind::Spawn,
5405    )
5406}
5407
5408fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5409    if let Some(reply) = runtime
5410        .deferred_reload_reply
5411        .lock()
5412        .unwrap_or_else(|p| p.into_inner())
5413        .take()
5414    {
5415        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5416            module_id: module_id.to_string(),
5417            reason: reason.to_string(),
5418        }));
5419    }
5420}
5421
5422fn schedule_respawn(
5423    runtime: &SupervisorRuntimeConfig,
5424    snapshot: &SharedSnapshot,
5425    module_id: &str,
5426    delay: Duration,
5427    kind: RespawnKind,
5428) -> Result<(), SuperviseError> {
5429    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5430    update_snapshot(snapshot, Some(module_id), |state| {
5431        state.respawn_pending = true
5432    })?;
5433    *runtime
5434        .scheduled_respawn
5435        .lock()
5436        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5437        deadline: Instant::now() + delay,
5438        kind,
5439    });
5440    Ok(())
5441}
5442
5443fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5444    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5445        state.health.last_action = Some(action);
5446        state.health.last_action_ms = Some(now_ms);
5447    });
5448}
5449
5450fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5451    match status {
5452        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5453        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5454        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5455    }
5456}
5457
5458/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5459/// returned to every `supervisor.list` and `supervisor.health` caller.
5460///
5461/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5462/// path: that request exists to return a module's complete metrics object, and
5463/// `ck health <module-id>` documents it as the way to see what the cached view
5464/// truncates. The asymmetry is the feature.
5465///
5466/// So a new caller must decide which side it is on rather than assume the cap is
5467/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5468/// the truncation that path exists to avoid.
5469fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5470    let metrics = metrics?;
5471    match serde_json::to_vec(&metrics) {
5472        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5473            "truncated": true,
5474            "original_bytes": encoded.len(),
5475        })),
5476        Ok(_) | Err(_) => Some(metrics),
5477    }
5478}
5479
5480/// Spread health probes so a fleet-wide restart does not converge them.
5481///
5482/// The delay is derived from the module id and probe index rather than a random
5483/// source, so it is deterministic per module: a module keeps its own offset
5484/// across daemon restarts instead of re-rolling into a collision.
5485fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5486    if cadence.is_zero() {
5487        return Duration::ZERO;
5488    }
5489    let cadence_ms = cadence.as_millis() as u64;
5490    // This early return is REDUNDANT, deliberately, and a mutation run will show
5491    // it surviving removal. Recording why here so the next person to notice does
5492    // not have to re-derive it:
5493    //
5494    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5495    //   a zero cadence and builds the Duration from whole milliseconds, so a
5496    //   sub-millisecond cadence cannot come from config.
5497    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5498    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5499    //   -- exactly what this returns.
5500    //
5501    // Kept as a guard against a future widening of the config parser (accepting
5502    // microseconds, say), which would make the sub-millisecond case reachable.
5503    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5504    // divides by zero. Remove this and nothing changes.
5505    if cadence_ms == 0 {
5506        return cadence;
5507    }
5508    // Note that this never returns less than one cadence, including for the FIRST
5509    // probe. So a freshly registered module reports health `unknown` for a full
5510    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5511    // ready to answer.
5512    //
5513    // That is a property of the supervisor's schedule, not of any module: an
5514    // operator watching a restart sees `unknown` and cannot tell it from a module
5515    // that is slow to warm. Measured on two unrelated modules, both flipping to
5516    // `ok` between 22s and 32s after restart.
5517    //
5518    // Left as-is because spreading the first probe is what keeps a fleet-wide
5519    // restart from firing fourteen simultaneous probes into a cold machine. The
5520    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5521    // that thundering herd for a faster first reading.
5522    let jitter_span = (cadence_ms / 10).max(1);
5523    let hash = module_id.as_bytes().iter().fold(
5524        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5525        |acc, byte| {
5526            acc.wrapping_mul(1099511628211)
5527                .wrapping_add(u64::from(*byte))
5528        },
5529    );
5530    cadence + Duration::from_millis(hash % jitter_span)
5531}
5532
5533#[cfg(test)]
5534mod tests {
5535    use super::*;
5536
5537    #[test]
5538    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5539        let handle = SupervisorHandle::new();
5540        let module_id = "readded-tombstone";
5541        handle.record_rescan_removal(module_id);
5542        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5543
5544        handle.apply_identity_configuration(&ModuleSpec {
5545            module_id: module_id.to_string(),
5546            program: PathBuf::from("/test/module"),
5547            args: Vec::new(),
5548            env: Vec::new(),
5549            reserved: false,
5550            reserved_prefixes: Vec::new(),
5551            protocol: ModuleProtocol::Subc,
5552            overlap: Default::default(),
5553        });
5554
5555        assert!(
5556            handle.removal_tombstone_age_ms(module_id).is_none(),
5557            "a re-added module must not retain a stale removal tombstone"
5558        );
5559    }
5560
5561    /// What one module's owner looked like from the control plane at the
5562    /// instant after its first process was spawned.
5563    #[derive(Debug, PartialEq, Eq)]
5564    struct OwnerInSpawnWindow {
5565        module_id: String,
5566        configured: bool,
5567        on_roster: bool,
5568        admission_refusal: Option<&'static str>,
5569    }
5570
5571    /// A supervised module's process can connect, register, sync its scopes
5572    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5573    /// the module on the roster until afterward; configured supervision does
5574    /// so before spawning. In both cases the owner must
5575    /// already read as configured, so a scoped `route.open` against it is
5576    /// refused as retryable `scope_not_synced` and not as terminal
5577    /// `scope_not_live` ("will never sync").
5578    ///
5579    /// The hook runs just after each initial spawn attempt, before returning to
5580    /// the caller, so no race with a real child's registration is needed.
5581    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5582    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5583        use crate::scopes::ScopeTable;
5584        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5585
5586        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5587        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5588            module_id: module_id.to_string(),
5589            program,
5590            args: Vec::new(),
5591            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5592                .into_iter()
5593                .map(|key| (key.to_string(), dir.path().display().to_string()))
5594                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5595                .collect(),
5596            reserved: false,
5597            reserved_prefixes: Vec::new(),
5598            protocol: ModuleProtocol::Subc,
5599            overlap: Default::default(),
5600        };
5601        let live = super::terminal_history_tests::fake_aft_stub_path();
5602        let missing = dir.path().join("definitely-missing-module");
5603
5604        let handle = SupervisorHandle::new();
5605        let mut supervisor =
5606            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5607                .with_handle(handle.clone());
5608        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5609        let hook_handle = handle.clone();
5610        let hook_observed = Arc::clone(&observed);
5611        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5612            // Exactly what the control plane computes for a scoped route.open
5613            // naming this module as the owner of a scope it has not synced.
5614            let configured = hook_handle.is_configured(module_id);
5615            let selector = ScopeSelector::new(
5616                Principal::Reserved {
5617                    module_id: module_id.to_string(),
5618                },
5619                "s".to_string(),
5620                Some(1),
5621            );
5622            let carrier = Principal::Reserved {
5623                module_id: "carrier".to_string(),
5624            };
5625            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5626                .admit(&carrier, module_id, &selector, configured)
5627            {
5628                Ok(_) => None,
5629                Err(refusal) => Some(refusal.code),
5630            };
5631            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5632                module_id: module_id.to_string(),
5633                configured,
5634                on_roster: hook_handle.get(module_id).is_some(),
5635                admission_refusal,
5636            });
5637        })));
5638
5639        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5640        let configured = supervisor
5641            .supervise_configured(stub("configured", live.clone()), true)
5642            .unwrap();
5643        let with_health = supervisor
5644            .supervise_configured_with_health(
5645                stub("with-health", live.clone()),
5646                true,
5647                HealthConfig::default(),
5648                None,
5649                RestartPolicy::default(),
5650            )
5651            .unwrap();
5652        // The failed-spawn path still puts the module on the roster (as
5653        // failed), so it is configured throughout.
5654        let failed = supervisor
5655            .supervise_configured_with_health(
5656                stub("failed-spawn", missing.clone()),
5657                true,
5658                HealthConfig::default(),
5659                None,
5660                RestartPolicy::default(),
5661            )
5662            .unwrap();
5663        // A failed plain `spawn` puts nothing on the roster, so its mark is
5664        // taken back once the spawn has failed.
5665        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5666
5667        let expected = [
5668            "plain",
5669            "configured",
5670            "with-health",
5671            "failed-spawn",
5672            "spawn-error",
5673        ]
5674        .into_iter()
5675        .map(|module_id| OwnerInSpawnWindow {
5676            module_id: module_id.to_string(),
5677            configured: true,
5678            on_roster: !matches!(module_id, "plain" | "spawn-error"),
5679            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5680        })
5681        .collect::<Vec<_>>();
5682        assert_eq!(*observed.lock().unwrap(), expected);
5683
5684        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5685            assert!(
5686                handle.get(module_id).is_some(),
5687                "{module_id} is on the roster"
5688            );
5689            assert!(
5690                handle.is_configured(module_id),
5691                "{module_id} stays configured"
5692            );
5693        }
5694        assert!(handle.get("spawn-error").is_none());
5695        assert!(
5696            !handle.is_configured("spawn-error"),
5697            "a plain spawn that failed must not leave its module marked configured"
5698        );
5699
5700        // Leaving the roster clears the mark with it.
5701        handle.retire("failed-spawn");
5702        assert!(!handle.is_configured("failed-spawn"));
5703
5704        for module in [plain, configured, with_health] {
5705            module.stop().await.unwrap();
5706        }
5707        drop(failed);
5708    }
5709
5710    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5711        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5712        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5713            snapshot.process_alive = true;
5714            snapshot.pid = Some(41);
5715            snapshot.spawned_at_ms = Some(42);
5716            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5717            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5718            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5719                device: 43,
5720                inode: 44,
5721                #[cfg(windows)]
5722                windows_image: None,
5723            });
5724        })
5725        .unwrap();
5726        snapshot
5727    }
5728
5729    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5730        let snapshot = lock_snapshot(snapshot).unwrap();
5731        assert!(!snapshot.process_alive);
5732        assert_eq!(snapshot.pid, None);
5733        assert_eq!(snapshot.spawned_at_ms, None);
5734        assert_eq!(snapshot.spawned_from, None);
5735        assert_eq!(snapshot.launch_spec_digest, None);
5736        assert_eq!(snapshot.spawned_file_identity, None);
5737    }
5738
5739    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5740    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5741        let supervisor =
5742            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5743        let mut runtime = supervisor.runtime_config();
5744        runtime.test_seed_stale_facts_before_enable_spawn = true;
5745        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5746        let mut child = None;
5747        let spec = ModuleSpec {
5748            module_id: "failed-enable-clears-facts".to_string(),
5749            program: PathBuf::from("/definitely/missing/failed-enable-module"),
5750            args: Vec::new(),
5751            env: Vec::new(),
5752            reserved: false,
5753            reserved_prefixes: Vec::new(),
5754            protocol: ModuleProtocol::Subc,
5755            overlap: Default::default(),
5756        };
5757
5758        let result = set_child_enabled(
5759            &spec,
5760            &runtime,
5761            &supervisor.registry,
5762            &supervisor.process_liveness,
5763            &snapshot,
5764            &mut child,
5765            true,
5766        )
5767        .await;
5768
5769        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5770        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5771        assert_snapshot_process_facts_cleared(&snapshot);
5772    }
5773
5774    #[tokio::test]
5775    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5776        let supervisor =
5777            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5778        let runtime = supervisor.runtime_config();
5779        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5780            ModuleState::Restarting,
5781            true,
5782        )));
5783        let spec = ModuleSpec {
5784            module_id: "start-stranded-restarting".to_string(),
5785            program: super::terminal_history_tests::fake_aft_stub_path(),
5786            args: Vec::new(),
5787            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5788            reserved: false,
5789            reserved_prefixes: Vec::new(),
5790            protocol: ModuleProtocol::None,
5791            overlap: Default::default(),
5792        };
5793        let mut child = None;
5794        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5795        assert!(!super::set_child_enabled(
5796            &spec,
5797            &runtime,
5798            &Registry::default(),
5799            &supervisor.process_liveness,
5800            &snapshot,
5801            &mut child,
5802            true
5803        )
5804        .await
5805        .unwrap());
5806        assert!(child.is_none());
5807        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5808        assert!(super::set_child_enabled(
5809            &spec,
5810            &runtime,
5811            &Registry::default(),
5812            &supervisor.process_liveness,
5813            &snapshot,
5814            &mut child,
5815            true
5816        )
5817        .await
5818        .unwrap());
5819        assert_eq!(
5820            lock_snapshot(&snapshot).unwrap().state,
5821            ModuleState::Running
5822        );
5823        let mut child = child.unwrap();
5824        child.start_kill().unwrap();
5825        child.wait().await.unwrap();
5826    }
5827
5828    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5829    async fn failed_reload_spawn_clears_current_process_facts() {
5830        let supervisor =
5831            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5832        let mut runtime = supervisor.runtime_config();
5833        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5834        let snapshot = stale_process_snapshot(ModuleState::Running, true);
5835        let mut child = None;
5836        let spec = ModuleSpec {
5837            module_id: "failed-reload-clears-facts".to_string(),
5838            program: PathBuf::from("/unused/failed-reload-module"),
5839            args: Vec::new(),
5840            env: Vec::new(),
5841            reserved: false,
5842            reserved_prefixes: Vec::new(),
5843            protocol: ModuleProtocol::Subc,
5844            overlap: Default::default(),
5845        };
5846
5847        let result = handle_reload_spawn_failure(
5848            &spec,
5849            &runtime,
5850            &supervisor.process_liveness,
5851            &snapshot,
5852            &mut child,
5853            "forced reload spawn failure".to_string(),
5854        )
5855        .await;
5856
5857        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5858        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5859        assert_snapshot_process_facts_cleared(&snapshot);
5860    }
5861
5862    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5863    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5864        let supervisor =
5865            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5866        let snapshot = stale_process_snapshot(ModuleState::Running, true);
5867        let module = supervisor.supervised_module(
5868            ModuleSpec {
5869                module_id: "drop-clears-facts".to_string(),
5870                program: PathBuf::from("/unused/drop-module"),
5871                args: Vec::new(),
5872                env: Vec::new(),
5873                reserved: false,
5874                reserved_prefixes: Vec::new(),
5875                protocol: ModuleProtocol::Subc,
5876                overlap: Default::default(),
5877            },
5878            supervisor.runtime_config(),
5879            Arc::clone(&snapshot),
5880            None,
5881        );
5882        assert!(!module
5883            .inner
5884            .monitor
5885            .lock()
5886            .unwrap()
5887            .as_ref()
5888            .unwrap()
5889            .is_finished());
5890
5891        drop(module);
5892
5893        assert_eq!(
5894            lock_snapshot(&snapshot).unwrap().state,
5895            ModuleState::Stopped
5896        );
5897        assert_snapshot_process_facts_cleared(&snapshot);
5898    }
5899
5900    #[cfg(unix)]
5901    #[tokio::test]
5902    async fn rescan_preserves_running_protocol_until_respawn() {
5903        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5904        let initial = ModuleSpec {
5905            module_id: "rescan-protocol".into(),
5906            program: PathBuf::from("/bin/sleep"),
5907            args: vec!["60".into()],
5908            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5909                .into_iter()
5910                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
5911                .collect(),
5912            reserved: false,
5913            reserved_prefixes: vec![],
5914            protocol: ModuleProtocol::None,
5915            overlap: Default::default(),
5916        };
5917        let supervisor =
5918            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5919        let module = supervisor.spawn(initial.clone()).unwrap();
5920        assert!(module.status().unwrap().live);
5921        let mut next = initial;
5922        next.protocol = ModuleProtocol::Subc;
5923        module
5924            .update_configuration(next.clone(), HealthConfig::default(), None)
5925            .await
5926            .unwrap();
5927        assert!(
5928            module.status().unwrap().live,
5929            "rescan must not require HELLO from the old non-wire process"
5930        );
5931        let runtime = supervisor.runtime_config();
5932        let action = on_child_exit(
5933            &next,
5934            RestartPolicy::default(),
5935            &supervisor.registry,
5936            &module.inner.snapshot,
5937            &runtime.terminal_ring,
5938            &runtime.spawn_events,
5939            &runtime.child_roster,
5940            ExitReport {
5941                kind: ExitKind::Clean,
5942                code: Some(0),
5943                signal: None,
5944                at_ms: unix_ms_now(),
5945            },
5946        )
5947        .await;
5948        assert!(
5949            matches!(action, NextAction::Restart { .. }),
5950            "the old non-wire process's clean exit must restart"
5951        );
5952        module.drain().await.unwrap();
5953    }
5954
5955    #[cfg(unix)]
5956    #[tokio::test]
5957    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
5958        use std::os::unix::fs::PermissionsExt;
5959        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
5960        let script = dir.join("module.sh");
5961        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
5962        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
5963        let record_path = dir.join("live-children.json");
5964        let supervisor =
5965            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5966                .with_live_children_record(&record_path);
5967        for (program, args) in [
5968            (PathBuf::from("sleep"), vec!["60".into()]),
5969            (script, vec![]),
5970        ] {
5971            let spec = ModuleSpec {
5972                module_id: "image-identity".into(),
5973                program: program.clone(),
5974                args,
5975                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5976                    .into_iter()
5977                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
5978                    .collect(),
5979                reserved: false,
5980                reserved_prefixes: vec![],
5981                protocol: ModuleProtocol::None,
5982                overlap: Default::default(),
5983            };
5984            let module = supervisor.spawn(spec).unwrap();
5985            #[cfg(target_os = "macos")]
5986            {
5987                // SETEXEC confirmation is asynchronous; the orphan record must
5988                // identify the final image, never the intermediate trampoline.
5989                let deadline = Instant::now() + Duration::from_secs(5);
5990                while crate::live_children::read_record(&record_path)
5991                    .unwrap()
5992                    .iter()
5993                    .all(|entry| entry.executable.is_none())
5994                {
5995                    assert!(Instant::now() < deadline, "module image was not confirmed");
5996                    tokio::time::sleep(Duration::from_millis(5)).await;
5997                }
5998            }
5999            let entry = crate::live_children::read_record(&record_path)
6000                .unwrap()
6001                .pop()
6002                .unwrap();
6003            let observed = subc_os::Process::open(entry.pid)
6004                .unwrap()
6005                .unwrap()
6006                .observe()
6007                .unwrap();
6008            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6009            module.drain().await.unwrap();
6010            assert_eq!(
6011                verdict,
6012                crate::live_children::IdentityVerdict::Matches,
6013                "program {program:?}: recorded {entry:?}, observed {observed:?}"
6014            );
6015        }
6016    }
6017
6018    #[cfg(unix)]
6019    fn http_fixture(
6020        dir: &std::path::Path,
6021        url: &str,
6022        threshold: u32,
6023    ) -> crate::daemon_config::ConfiguredModule {
6024        let path = dir.join("subc.jsonc");
6025        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6026            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6027            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6028            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6029            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6030        }}}).to_string()).unwrap();
6031        crate::daemon_config::load(&path)
6032            .unwrap()
6033            .unwrap()
6034            .modules
6035            .pop()
6036            .unwrap()
6037    }
6038
6039    #[cfg(unix)]
6040    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6041        timeout(Duration::from_secs(5), async {
6042            loop {
6043                if module.status().unwrap().health.status == status {
6044                    break;
6045                }
6046                sleep(Duration::from_millis(5)).await;
6047            }
6048        })
6049        .await
6050        .unwrap_or_else(|_| {
6051            panic!(
6052                "expected {status:?}, got {:?}",
6053                module.status().unwrap().health
6054            )
6055        });
6056    }
6057
6058    #[cfg(unix)]
6059    #[tokio::test]
6060    async fn http_health_status_flips_ok_failing_ok() {
6061        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6062        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6063        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6064        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6065        let serving_status = status.clone();
6066        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6067        let server = tokio::spawn(async move {
6068            loop {
6069                let (mut stream, _) = listener.accept().await.unwrap();
6070                let mut request = [0u8; 2048];
6071                let count = stream.read(&mut request).await.unwrap();
6072                assert!(count > 0, "a probe must send an HTTP request");
6073                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6074                let body = if code == 200 {
6075                    "ready"
6076                } else {
6077                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6078                };
6079                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6080                let _ = stream.write_all(response.as_bytes()).await;
6081            }
6082        });
6083        let configured = http_fixture(&dir, &url, 1000);
6084        let module =
6085            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6086                .supervise_configured_with_health(
6087                    configured.module_spec(),
6088                    true,
6089                    configured.health,
6090                    configured.drain_timeout_ms,
6091                    configured.restart,
6092                )
6093                .unwrap();
6094        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6095        status.store(503, std::sync::atomic::Ordering::SeqCst);
6096        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6097        assert!(module
6098            .status()
6099            .unwrap()
6100            .health
6101            .detail
6102            .unwrap()
6103            .contains("scratch failure"));
6104        status.store(200, std::sync::atomic::Ordering::SeqCst);
6105        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6106        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6107        let before = module.status().unwrap();
6108        let (spec, mut health) = module.configuration().unwrap();
6109        health.http = None;
6110        module
6111            .update_configuration(spec.clone(), health.clone(), Some(10))
6112            .await
6113            .unwrap();
6114        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6115        health.http = Some(url);
6116        module
6117            .update_configuration(spec, health, Some(10))
6118            .await
6119            .unwrap();
6120        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6121        assert_eq!(
6122            module.status().unwrap().pid,
6123            before.pid,
6124            "changing a probe must apply live, not restart its process"
6125        );
6126        let (spec, mut health) = module.configuration().unwrap();
6127        health.failure_threshold = 2;
6128        module
6129            .update_configuration(spec, health, Some(10))
6130            .await
6131            .unwrap();
6132        status.store(503, std::sync::atomic::Ordering::SeqCst);
6133        timeout(Duration::from_secs(5), async {
6134            while module.status().unwrap().spawn_generation == before.spawn_generation {
6135                sleep(Duration::from_millis(5)).await;
6136            }
6137        })
6138        .await
6139        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6140        module.drain().await.unwrap();
6141        server.abort();
6142    }
6143
6144    #[cfg(unix)]
6145    #[tokio::test]
6146    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6147        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6148        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6149        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6150        drop(unused);
6151        let configured = http_fixture(&dir, &url, 2);
6152        let module =
6153            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6154                .supervise_configured_with_health(
6155                    configured.module_spec(),
6156                    true,
6157                    configured.health,
6158                    configured.drain_timeout_ms,
6159                    configured.restart,
6160                )
6161                .unwrap();
6162        let before = module.status().unwrap().spawn_generation;
6163        timeout(Duration::from_secs(5), async {
6164            loop {
6165                let status = module.status().unwrap();
6166                if status.spawn_generation > before {
6167                    assert!(status.lifetime_restarts > 0);
6168                    break;
6169                }
6170                sleep(Duration::from_millis(5)).await;
6171            }
6172        })
6173        .await
6174        .expect("sustained HTTP refusal must trigger the health restart policy");
6175        module.drain().await.unwrap();
6176    }
6177
6178    #[cfg(unix)]
6179    #[tokio::test]
6180    async fn http_health_timeout_honours_deadline() {
6181        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6182        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6183        let server = tokio::spawn(async move {
6184            let _held = listener.accept().await.unwrap();
6185            std::future::pending::<()>().await;
6186        });
6187        let error = timeout(
6188            Duration::from_secs(1),
6189            probe_http_health(&url, Duration::from_millis(10)),
6190        )
6191        .await
6192        .expect("the probe must enforce its own deadline")
6193        .unwrap_err();
6194        server.abort();
6195        assert!(error.to_string().contains("timed out"));
6196    }
6197
6198    #[cfg(unix)]
6199    #[tokio::test]
6200    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6201        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6202        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6203        let url = format!(
6204            "http://localhost:{}/healthz",
6205            listener.local_addr().unwrap().port()
6206        );
6207        let server = tokio::spawn(async move {
6208            let (mut stream, _) = listener.accept().await.unwrap();
6209            let mut request = [0u8; 2048];
6210            assert!(stream.read(&mut request).await.unwrap() > 0);
6211            stream
6212                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6213                .await
6214                .unwrap();
6215        });
6216        // The deadline only bounds a hang. A probe that never tried the IPv6
6217        // address would be refused on 127.0.0.1 and fail at once, so a longer
6218        // deadline does not weaken the assertion; one second timed out under a
6219        // loaded parallel test run.
6220        assert_eq!(
6221            probe_http_health(&url, Duration::from_secs(10))
6222                .await
6223                .unwrap()
6224                .status,
6225            HealthStatus::Ok
6226        );
6227        server.await.unwrap();
6228    }
6229
6230    #[cfg(unix)]
6231    #[tokio::test]
6232    async fn http_health_timeout_keeps_partial_status_and_body() {
6233        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6234        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6235        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6236        let server = tokio::spawn(async move {
6237            let (mut stream, _) = listener.accept().await.unwrap();
6238            let mut request = [0u8; 2048];
6239            assert!(stream.read(&mut request).await.unwrap() > 0);
6240            stream
6241                .write_all(
6242                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6243                )
6244                .await
6245                .unwrap();
6246            std::future::pending::<()>().await;
6247        });
6248        let error = probe_http_health(&url, Duration::from_secs(1))
6249            .await
6250            .unwrap_err()
6251            .to_string();
6252        server.abort();
6253        assert!(
6254            error.contains("timed out")
6255                && error.contains("503 Unavailable")
6256                && error.contains("partial diagnostic"),
6257            "{error}"
6258        );
6259    }
6260
6261    #[cfg(unix)]
6262    #[tokio::test]
6263    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6264        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6265        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6266        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6267        let server = tokio::spawn(async move {
6268            let (mut stream, _) = listener.accept().await.unwrap();
6269            let mut request = [0u8; 2048];
6270            assert!(stream.read(&mut request).await.unwrap() > 0);
6271            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6272            let response = format!(
6273                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6274                body.len()
6275            );
6276            stream.write_all(response.as_bytes()).await.unwrap();
6277        });
6278        let error = probe_http_health(&url, Duration::from_secs(1))
6279            .await
6280            .unwrap_err()
6281            .to_string();
6282        server.await.unwrap();
6283        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6284        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6285        assert!(!error.contains("not-in-diagnostic"));
6286    }
6287
6288    #[cfg(unix)]
6289    #[tokio::test]
6290    async fn http_health_real_nats_server_monitoring() {
6291        if std::process::Command::new("nats-server")
6292            .arg("--version")
6293            .env("XDG_DATA_HOME", std::env::temp_dir())
6294            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6295            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6296            .output()
6297            .is_err()
6298        {
6299            eprintln!(
6300                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6301            );
6302            return;
6303        }
6304        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6305        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6306        let port = monitor.local_addr().unwrap().port();
6307        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6308        let client_port = client.local_addr().unwrap().port();
6309        let config = dir.join("server.conf");
6310        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6311        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6312        configured.program = PathBuf::from("nats-server");
6313        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6314        drop(monitor);
6315        drop(client);
6316        let module =
6317            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6318                .supervise_configured_with_health(
6319                    configured.module_spec(),
6320                    true,
6321                    configured.health,
6322                    configured.drain_timeout_ms,
6323                    configured.restart,
6324                )
6325                .unwrap();
6326        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6327        module.drain().await.unwrap();
6328    }
6329
6330    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6331    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6332        let supervisor =
6333            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6334        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6335        let initial = ModuleSpec {
6336            module_id: "rescan-preserves-spawn-facts".to_string(),
6337            program: PathBuf::from("/spawned/module"),
6338            args: Vec::new(),
6339            env: Vec::new(),
6340            reserved: false,
6341            reserved_prefixes: Vec::new(),
6342            protocol: ModuleProtocol::Subc,
6343            overlap: Default::default(),
6344        };
6345        let module = supervisor.supervised_module(
6346            initial.clone(),
6347            supervisor.runtime_config(),
6348            snapshot,
6349            None,
6350        );
6351        let before = module.status().unwrap();
6352        let mut replacement = initial;
6353        replacement.program = PathBuf::from("/rescanned/replacement-module");
6354
6355        module
6356            .update_configuration(replacement, HealthConfig::default(), None)
6357            .await
6358            .unwrap();
6359
6360        let after = module.status().unwrap();
6361        assert_eq!(after.pid, before.pid);
6362        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6363        assert_eq!(after.spawned_from, before.spawned_from);
6364        drop(module);
6365    }
6366}
6367
6368fn unix_ms_now() -> u64 {
6369    SystemTime::now()
6370        .duration_since(UNIX_EPOCH)
6371        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6372        .unwrap_or(0)
6373}
6374
6375async fn supervise_loop(
6376    mut spec: ModuleSpec,
6377    mut runtime: SupervisorRuntimeConfig,
6378    registry: Arc<Registry>,
6379    process_liveness: Arc<SupervisorProcessLiveness>,
6380    snapshot: SharedSnapshot,
6381    mut child: Option<SupervisedChild>,
6382    mut commands: mpsc::Receiver<SupervisorCommand>,
6383) {
6384    let mut health_probe = HealthProbeRuntime::default();
6385    // Registry writes (including embedded callers) notify this module only.
6386    // Subscribe before the first refresh; watch retains changes that arrive
6387    // while commands or probes are running. No polling fallback is needed.
6388    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6389        Ok(changes) => Some(changes),
6390        Err(err) => {
6391            // A poisoned registry cannot accept further writes, so it cannot
6392            // recover via a timer. Keep exit and command handling alive.
6393            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6394            None
6395        }
6396    };
6397    // All restart backoffs run here, including health and operator requests.
6398    // While one is pending the loop serves commands, so disable or drain can
6399    // cancel the replacement without spawning a process just to stop it.
6400    let mut pending_respawn: Option<PendingRespawn> = None;
6401    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
6402    // before anything else so a stop that interrupted a swap runs at once.
6403    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6404    let mut previous_instance_recheck = tokio::time::interval_at(
6405        Instant::now() + Duration::from_secs(5),
6406        Duration::from_secs(5),
6407    );
6408    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6409    loop {
6410        #[cfg(test)]
6411        {
6412            lock_snapshot(&snapshot).unwrap().actor_select = None;
6413        }
6414        #[cfg(target_os = "macos")]
6415        if let Some(active) = child.as_mut() {
6416            active.confirm_privacy_exec().await;
6417        }
6418        if let Some(scheduled) = runtime
6419            .scheduled_respawn
6420            .lock()
6421            .unwrap_or_else(|p| p.into_inner())
6422            .take()
6423        {
6424            pending_respawn = Some(scheduled);
6425        }
6426        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6427            pending_respawn = None;
6428            cancel_deferred_reload(
6429                &runtime,
6430                &spec.module_id,
6431                "respawn cancelled by a supervisor command",
6432            );
6433        }
6434        if child.is_none()
6435            && pending_respawn.is_none()
6436            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6437        {
6438            cancel_deferred_reload(
6439                &runtime,
6440                &spec.module_id,
6441                "respawn cancelled before a replacement was spawned",
6442            );
6443            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6444                state.respawn_pending = false;
6445                state.coalesced_restart_pending = false;
6446                if matches!(
6447                    state.state,
6448                    ModuleState::Restarting
6449                        | ModuleState::Starting
6450                        | ModuleState::Draining
6451                        | ModuleState::Unresponsive
6452                ) {
6453                    error!(module_id = %spec.module_id, state = ?state.state,
6454                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6455                    state.state = ModuleState::Failed;
6456                    clear_current_process_facts(state);
6457                }
6458            });
6459        }
6460        if let Some(command) = requeued.pop_front() {
6461            if !handle_supervisor_command(
6462                command,
6463                &mut spec,
6464                &mut runtime,
6465                &registry,
6466                &process_liveness,
6467                &snapshot,
6468                &mut child,
6469                &mut commands,
6470                &mut requeued,
6471            )
6472            .await
6473            {
6474                return;
6475            }
6476            if child.is_some() || !respawn_still_pending(&snapshot) {
6477                pending_respawn = None;
6478                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6479                    state.respawn_pending = false
6480                });
6481            }
6482            continue;
6483        }
6484        if child.is_some() {
6485            #[cfg(test)]
6486            {
6487                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6488            }
6489            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
6490            let wake_after = health_probe.wake_after();
6491            let probe_sleep = async move {
6492                match wake_after {
6493                    Some(delay) => sleep(delay).await,
6494                    None => std::future::pending().await,
6495                }
6496            };
6497            tokio::pin!(probe_sleep);
6498            // A `protocol: "none"` child (a plain process that never registers
6499            // over the subc wire, such as nats-server) with no HTTP health
6500            // check has nothing to probe, so it waits only for its exit or a
6501            // supervisor command. A rescan that adds an HTTP check is a command.
6502            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6503            #[cfg(test)]
6504            {
6505                let registration_pending = wire_child
6506                    && registration_changes
6507                        .as_ref()
6508                        .is_some_and(|changes| changes.has_changed().unwrap());
6509                let mut state = lock_snapshot(&snapshot).unwrap();
6510                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6511                    generation: state.spawn_generation,
6512                    turn: state.actor_turns,
6513                    registered_connection: health_probe.registered_connection,
6514                    next_probe_at: health_probe.next_probe_at,
6515                    wake_after,
6516                });
6517            }
6518            let active_child = child.as_mut().expect("child checked above");
6519            tokio::select! {
6520                wait_result = active_child.wait() => {
6521                    #[cfg(test)]
6522                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6523                    // Every arm below that gives up on the CHILD must keep the
6524                    // supervision task itself alive (child = None, loop
6525                    // continues into command-serving mode). Returning here
6526                    // closes the command channel, which makes the module
6527                    // permanently unrestartable in-band: a clean child exit
6528                    // of an enabled module once wedged the fleet this way
6529                    // ('supervisor command channel is closed') and required a
6530                    // full daemon restart to recover.
6531                    let exit_report = match wait_result {
6532                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6533                        Err(err) => {
6534                            active_child.drain_stderr(&spec.module_id).await;
6535                            fail_after_wait_error(
6536                                &spec.module_id,
6537                                &snapshot,
6538                                &runtime.terminal_ring,
6539                                &runtime.spawn_events,
6540                                format!("failed to wait for supervised module: {err}"),
6541                            );
6542                            untrack_if_registration_released(
6543                                &process_liveness,
6544                                &registry,
6545                                &spec.module_id,
6546                                &snapshot,
6547                            );
6548                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6549                            child = None;
6550                            continue;
6551                        }
6552                    };
6553                    active_child.drain_stderr(&spec.module_id).await;
6554
6555                    let next = on_child_exit(
6556                        &spec,
6557                        runtime.restart_policy,
6558                        &registry,
6559                        &snapshot,
6560                        &runtime.terminal_ring,
6561                        &runtime.spawn_events,
6562                        &runtime.child_roster,
6563                        exit_report,
6564                    ).await;
6565                    runtime.start_requester = None;
6566                    runtime.pending_activity_restart = false;
6567                    // The exit is recorded, so a daemon shutdown may stop
6568                    // waiting for this child (see `SupervisedChild::wait`).
6569                    active_child.release_roster();
6570                    match next {
6571                        NextAction::Stop { registration_released } => {
6572                            if registration_released {
6573                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6574                            }
6575                            child = None;
6576                        }
6577                        NextAction::Restart { schedule } => {
6578                            let delay = schedule.map_or(
6579                                runtime.restart_policy.delay_for_restart(0),
6580                                |schedule| schedule.delay,
6581                            );
6582                            if let Some(schedule) = schedule {
6583                                log_crash_respawn(&spec.module_id, schedule);
6584                            }
6585                            // The exited child is fully recorded at this point,
6586                            // so release it and count the backoff down in the
6587                            // command-serving branch below rather than sleeping
6588                            // here: commands cannot be received from inside this
6589                            // select arm, and an operator disable or drain that
6590                            // arrives during the backoff must cancel the pending
6591                            // respawn instead of waiting for it to spawn first.
6592                            child = None;
6593                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6594                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6595                        }
6596                    }
6597                }
6598                command = commands.recv() => {
6599                    #[cfg(test)]
6600                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6601                    let Some(command) = command else {
6602                        return;
6603                    };
6604                    if !handle_supervisor_command(
6605                        command,
6606                        &mut spec,
6607                        &mut runtime,
6608                        &registry,
6609                        &process_liveness,
6610                        &snapshot,
6611                        &mut child,
6612                        &mut commands,
6613                        &mut requeued,
6614                    ).await {
6615                        return;
6616                    }
6617                }
6618                _ = async {
6619                    match registration_changes.as_mut() {
6620                        Some(changes) => { let _ = changes.changed().await; }
6621                        None => std::future::pending().await,
6622                    }
6623                }, if wire_child => {
6624                    #[cfg(test)]
6625                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6626                }
6627                _ = &mut probe_sleep => {
6628                    #[cfg(test)]
6629                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6630                    if health_probe.due() {
6631                        run_health_probe_cycle(
6632                            &spec,
6633                            &runtime,
6634                            &registry,
6635                            &process_liveness,
6636                            &snapshot,
6637                            &mut child,
6638                        ).await;
6639                        if child.is_none() {
6640                            runtime.start_requester = None;
6641                            runtime.pending_activity_restart = false;
6642                        }
6643                        if child.is_some() {
6644                            health_probe.schedule_next(&spec, runtime.health.cadence);
6645                        }
6646                    }
6647                }
6648            }
6649        } else if let Some(pending) = pending_respawn {
6650            tokio::select! {
6651                _ = sleep_until(pending.deadline) => {
6652                    pending_respawn = None;
6653                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6654                    // A command handled below while the backoff elapsed may
6655                    // have stopped the module; never respawn past an operator's
6656                    // disable or drain.
6657                    if !respawn_still_pending(&snapshot) {
6658                        continue;
6659                    }
6660                    // The daemon began shutting down during the backoff: the
6661                    // spawn would be refused anyway, and refusing it here
6662                    // leaves the module stopped instead of reporting a
6663                    // failed restart.
6664                    if runtime.child_roster.is_closed() {
6665                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6666                            state.state = ModuleState::Stopped;
6667                        });
6668                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6669                        continue;
6670                    }
6671                    if let Err(err) = release_dead_registration(
6672                        &registry,
6673                        runtime.forwarding.as_deref(),
6674                        &snapshot,
6675                        &spec.module_id,
6676                    ).await {
6677                        fail_snapshot(&snapshot, Some(&spec.module_id), None);
6678                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6679                        continue;
6680                    }
6681
6682                    if matches!(pending.kind, RespawnKind::Reload) {
6683                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6684                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
6685                        if result.is_ok() && runtime.pending_activity_restart {
6686                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6687                        }
6688                        runtime.pending_activity_restart = false;
6689                        if let Some(reply) = reply { let _ = reply.send(result); }
6690                        continue;
6691                    }
6692                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6693                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6694                        Ok(next_child) => {
6695                            child = Some(next_child);
6696                            if runtime.pending_activity_restart {
6697                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6698                                runtime.pending_activity_restart = false;
6699                            }
6700                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6701                        }
6702                        Err(err) => {
6703                            fail_snapshot(&snapshot, Some(&spec.module_id), None);
6704                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6705                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6706                        }
6707                    }
6708                }
6709                command = commands.recv() => {
6710                    let Some(command) = command else {
6711                        return;
6712                    };
6713                    if !handle_supervisor_command(
6714                        command,
6715                        &mut spec,
6716                        &mut runtime,
6717                        &registry,
6718                        &process_liveness,
6719                        &snapshot,
6720                        &mut child,
6721                        &mut commands,
6722                        &mut requeued,
6723                    ).await {
6724                        return;
6725                    }
6726                    // Reconcile the pending respawn with what the command did:
6727                    // a start may already have spawned a fresh child,
6728                    // while a disable or drain moved the snapshot out of the
6729                    // state the respawn was counting down from.
6730                    if child.is_some() || !respawn_still_pending(&snapshot) {
6731                        pending_respawn = None;
6732                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6733                    }
6734                }
6735            }
6736        } else {
6737            let command = if lock_snapshot(&snapshot)
6738                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
6739            {
6740                tokio::select! {
6741                    command = commands.recv() => command,
6742                    _ = previous_instance_recheck.tick() => {
6743                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
6744                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
6745                            if enabled && !runtime.child_roster.is_closed() {
6746                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6747                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6748                                    Ok(next_child) => child = Some(next_child),
6749                                    Err(error) => {
6750                                        fail_snapshot(&snapshot, Some(&spec.module_id), None);
6751                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
6752                                    }
6753                                }
6754                            } else {
6755                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6756                                    state.waiting_for_previous_pid = None;
6757                                    state.waiting_for_previous_detail = None;
6758                                });
6759                            }
6760                        }
6761                        continue;
6762                    }
6763                }
6764            } else {
6765                commands.recv().await
6766            };
6767            let Some(command) = command else {
6768                return;
6769            };
6770            if !handle_supervisor_command(
6771                command,
6772                &mut spec,
6773                &mut runtime,
6774                &registry,
6775                &process_liveness,
6776                &snapshot,
6777                &mut child,
6778                &mut commands,
6779                &mut requeued,
6780            )
6781            .await
6782            {
6783                return;
6784            }
6785        }
6786    }
6787}
6788
6789fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
6790    info!(
6791        module_id,
6792        restart_in_window = schedule.restart_in_window,
6793        delay_ms = schedule.delay.as_millis() as u64,
6794        "respawning after crash"
6795    );
6796}
6797
6798/// Whether the respawn a backoff was counting down to is still wanted. A
6799/// disable or drain handled while the backoff elapsed moves the snapshot out
6800/// of `Restarting`, and the operator's stop must win over the pending respawn,
6801/// so every sleep-then-spawn path re-validates against the live snapshot
6802/// instead of assuming the state it left behind still holds.
6803fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
6804    matches!(
6805        lock_snapshot(snapshot),
6806        Ok(state) if state.enabled && state.state == ModuleState::Restarting
6807    )
6808}
6809
6810enum NextAction {
6811    Stop {
6812        registration_released: bool,
6813    },
6814    Restart {
6815        schedule: Option<CrashRestartSchedule>,
6816    },
6817}
6818
6819#[allow(clippy::too_many_arguments)]
6820async fn handle_supervisor_command(
6821    command: SupervisorCommand,
6822    spec: &mut ModuleSpec,
6823    runtime: &mut SupervisorRuntimeConfig,
6824    registry: &Arc<Registry>,
6825    process_liveness: &SupervisorProcessLiveness,
6826    snapshot: &SharedSnapshot,
6827    child: &mut Option<SupervisedChild>,
6828    commands: &mut mpsc::Receiver<SupervisorCommand>,
6829    requeued: &mut VecDeque<SupervisorCommand>,
6830) -> bool {
6831    let command =
6832        if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
6833            match command {
6834                SupervisorCommand::Restart { reply, .. }
6835                | SupervisorCommand::Reload { reply, .. }
6836                | SupervisorCommand::Swap { reply, .. } => {
6837                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6838                    return true;
6839                }
6840                SupervisorCommand::SetEnabled {
6841                    enabled: true,
6842                    reply,
6843                    ..
6844                } => {
6845                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6846                    return true;
6847                }
6848                other => other,
6849            }
6850        } else {
6851            command
6852        };
6853    match command {
6854        SupervisorCommand::Drain { requester, reply } => {
6855            // A plain stop runs no forwarding drain, so nothing reaches the
6856            // module over its connection before the wait: ask by signal.
6857            let result = drain_optional_child(
6858                &spec.module_id,
6859                spec.protocol,
6860                StopNotice::NotSent,
6861                registry,
6862                runtime.forwarding.as_deref(),
6863                snapshot,
6864                &runtime.terminal_ring,
6865                &runtime.spawn_events,
6866                child,
6867                runtime.drain_timeout,
6868                ModuleState::Stopped,
6869                None,
6870            )
6871            .await;
6872            let registration_released = result.is_ok();
6873            if registration_released {
6874                runtime.record_activity_transition(
6875                    "subc.module_stopped",
6876                    &spec.module_id,
6877                    requester,
6878                );
6879            }
6880            let _ = reply.send(result);
6881            if registration_released {
6882                process_liveness.untrack_if_current(&spec.module_id, snapshot);
6883            }
6884            false
6885        }
6886        SupervisorCommand::Retire { requester, reply } => {
6887            let result = async {
6888                let stop_notice = begin_forwarding_drain_if_configured(
6889                    spec,
6890                    runtime,
6891                    registry,
6892                    snapshot,
6893                    None,
6894                    RouteCloseReason::Disable,
6895                )
6896                .await?;
6897                drain_optional_child(
6898                    &spec.module_id,
6899                    spec.protocol,
6900                    stop_notice,
6901                    registry,
6902                    runtime.forwarding.as_deref(),
6903                    snapshot,
6904                    &runtime.terminal_ring,
6905                    &runtime.spawn_events,
6906                    child,
6907                    runtime.drain_timeout,
6908                    ModuleState::Stopped,
6909                    None,
6910                )
6911                .await
6912            }
6913            .await;
6914            let registration_released = result.is_ok();
6915            if registration_released {
6916                runtime.record_activity_transition(
6917                    "subc.module_stopped",
6918                    &spec.module_id,
6919                    requester,
6920                );
6921            }
6922            let _ = reply.send(result);
6923            if registration_released {
6924                process_liveness.untrack_if_current(&spec.module_id, snapshot);
6925            }
6926            false
6927        }
6928        SupervisorCommand::Restart {
6929            requester,
6930            drain_timeout_ms,
6931            received_at_generation,
6932            queued_at,
6933            reply,
6934        } => {
6935            // Without this line a restart that waited in the queue (behind a
6936            // health probe cycle or another command) was invisible: the log
6937            // showed only the drain timing out, minutes after the operator's call.
6938            info!(
6939                module_id = %spec.module_id,
6940                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
6941                "restart command dequeued"
6942            );
6943            // ACK AT INITIATION, not completion. The blocking form deadlocked any
6944            // caller whose own request lane rides the module being restarted: the
6945            // caller's in-flight request keeps the drain from quiescing, the drain
6946            // keeps the restart from completing, and the completion keeps the reply
6947            // from releasing the caller — so the drain always timed out and cut the
6948            // initiator with a GOODBYE, even on a healthy module. Replying once the
6949            // restart is validated lets a self-lane caller settle, which is exactly
6950            // what makes the drain succeed. Completion is observable via
6951            // supervisor.list / module status; a post-ack failure lands the module
6952            // in a visible terminal state below rather than in a reply nobody can
6953            // receive.
6954            let validation = match lock_snapshot(snapshot) {
6955                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
6956                    module_id: spec.module_id.clone(),
6957                }),
6958                Ok(_) => Ok(()),
6959                Err(err) => Err(err),
6960            };
6961            let initiated = validation.is_ok();
6962            let _ = reply.send(validation);
6963            // A restart asks for a fresh process. Commands run one at a time,
6964            // so a restart queued behind another restart (two operator calls
6965            // in quick succession) is dequeued the moment the first one has
6966            // spawned its replacement -- before that process has sent HELLO.
6967            // Running it would drain and kill the process the first restart
6968            // just produced, which is the opposite of what both callers asked
6969            // for. If a process spawned after this request was received is
6970            // still supervised, the request is already satisfied. Not when the
6971            // configuration changed since that spawn: then the newer process
6972            // predates the spec this restart may exist to apply.
6973            let satisfied_by_generation = if initiated && child.is_some() {
6974                lock_snapshot(snapshot).ok().and_then(|state| {
6975                    (state.spawn_generation > received_at_generation
6976                        && !state.configuration_updated_since_spawn)
6977                        .then_some(state.spawn_generation)
6978                })
6979            } else {
6980                None
6981            };
6982            let satisfied_by_pending = initiated
6983                && child.is_none()
6984                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
6985                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
6986                    if pending {
6987                        state.coalesced_restart_pending = true;
6988                    }
6989                    pending
6990                });
6991            if satisfied_by_pending {
6992                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
6993            } else if let Some(generation) = satisfied_by_generation {
6994                info!(
6995                    module_id = %spec.module_id,
6996                    received_at_generation,
6997                    "restart already satisfied by generation {generation}; not restarting again"
6998                );
6999            } else if initiated {
7000                // Precedence: this restart's operator override, else the module's
7001                // configured budget (already resolved into the runtime).
7002                let drain_timeout = drain_timeout_ms
7003                    .map(Duration::from_millis)
7004                    .unwrap_or(runtime.drain_timeout);
7005                if let Err(err) = restart_child(
7006                    spec,
7007                    runtime,
7008                    registry,
7009                    process_liveness,
7010                    snapshot,
7011                    child,
7012                    drain_timeout,
7013                )
7014                .await
7015                {
7016                    warn!(
7017                        module_id = %spec.module_id,
7018                        error = %err,
7019                        "operator restart failed after initiation ack; module state carries the outcome"
7020                    );
7021                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7022                        state.state = ModuleState::Failed;
7023                        clear_current_process_facts(state);
7024                    });
7025                } else {
7026                    runtime.start_requester = requester;
7027                    runtime.pending_activity_restart = true;
7028                }
7029            }
7030            true
7031        }
7032        SupervisorCommand::Reload { requester, reply } => {
7033            let result =
7034                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7035            if result.is_ok() {
7036                runtime.start_requester = requester;
7037                runtime.pending_activity_restart = true;
7038            }
7039            if result.is_ok()
7040                && runtime
7041                    .scheduled_respawn
7042                    .lock()
7043                    .unwrap_or_else(|p| p.into_inner())
7044                    .as_ref()
7045                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7046            {
7047                *runtime
7048                    .deferred_reload_reply
7049                    .lock()
7050                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7051            } else {
7052                let _ = reply.send(result);
7053            }
7054            true
7055        }
7056        SupervisorCommand::SetEnabled {
7057            enabled,
7058            requester,
7059            reply,
7060        } => {
7061            let previous_requester = runtime.start_requester.clone();
7062            if enabled {
7063                runtime.start_requester = requester.clone();
7064            }
7065            let result = set_child_enabled(
7066                spec,
7067                runtime,
7068                registry,
7069                process_liveness,
7070                snapshot,
7071                child,
7072                enabled,
7073            )
7074            .await;
7075            if matches!(result, Ok(true)) {
7076                runtime.pending_activity_restart = false;
7077                if enabled {
7078                    runtime.start_requester = requester;
7079                } else {
7080                    runtime.start_requester = None;
7081                    runtime.record_activity_transition(
7082                        "subc.module_disabled",
7083                        &spec.module_id,
7084                        requester,
7085                    );
7086                }
7087            } else {
7088                runtime.start_requester = previous_requester;
7089            }
7090            let _ = reply.send(result);
7091            true
7092        }
7093        SupervisorCommand::UpdateConfiguration {
7094            spec: next_spec,
7095            health,
7096            drain_timeout_ms,
7097            reply,
7098        } => {
7099            if let Some(handle) = &runtime.supervisor_handle {
7100                handle.apply_identity_configuration(&next_spec);
7101            }
7102            *spec = next_spec;
7103            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7104                state.configuration_updated_since_spawn = true;
7105            });
7106            let health_changed = runtime.health != health;
7107            runtime.health = health;
7108            // Reset the cadence and old endpoint's failure streak on a live
7109            // health-policy change rather than waiting for its old deadline.
7110            if health_changed {
7111                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7112                    state.health = ModuleHealthStatus::default();
7113                });
7114            }
7115            runtime.drain_timeout = drain_timeout_ms
7116                .map(Duration::from_millis)
7117                .unwrap_or(runtime.default_drain_timeout);
7118            *runtime
7119                .effective_drain_timeout
7120                .lock()
7121                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7122            let _ = reply.send(());
7123            true
7124        }
7125        SupervisorCommand::Swap {
7126            requester,
7127            ready_timeout,
7128            reply,
7129        } => {
7130            let mut swap_runtime = runtime.clone();
7131            swap_runtime.start_requester = requester;
7132            let before = lock_snapshot(snapshot)
7133                .ok()
7134                .map(|state| state.spawn_generation);
7135            let end = swap::run_swap(
7136                spec,
7137                &swap_runtime,
7138                registry,
7139                process_liveness,
7140                snapshot,
7141                child,
7142                commands,
7143                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7144                reply,
7145            )
7146            .await;
7147            if before
7148                != lock_snapshot(snapshot)
7149                    .ok()
7150                    .map(|state| state.spawn_generation)
7151            {
7152                runtime.start_requester = swap_runtime.start_requester;
7153            }
7154            requeued.extend(end.requeue);
7155            true
7156        }
7157    }
7158}
7159
7160async fn restart_child(
7161    spec: &ModuleSpec,
7162    runtime: &SupervisorRuntimeConfig,
7163    registry: &Registry,
7164    process_liveness: &SupervisorProcessLiveness,
7165    snapshot: &SharedSnapshot,
7166    child: &mut Option<SupervisedChild>,
7167    drain_timeout: Duration,
7168) -> Result<(), SuperviseError> {
7169    // Restart cycles a running module; it must not silently start a disabled one.
7170    if !lock_snapshot(snapshot)?.enabled {
7171        return Err(SuperviseError::Disabled {
7172            module_id: spec.module_id.clone(),
7173        });
7174    }
7175    let stop_notice = begin_forwarding_drain_with_timeout(
7176        spec,
7177        runtime,
7178        registry,
7179        snapshot,
7180        None,
7181        RouteCloseReason::Restart,
7182        drain_timeout,
7183    )
7184    .await?;
7185
7186    if child.is_some() {
7187        drain_optional_child(
7188            &spec.module_id,
7189            spec.protocol,
7190            stop_notice,
7191            registry,
7192            runtime.forwarding.as_deref(),
7193            snapshot,
7194            &runtime.terminal_ring,
7195            &runtime.spawn_events,
7196            child,
7197            drain_timeout,
7198            ModuleState::Restarting,
7199            Some(true),
7200        )
7201        .await?;
7202    } else {
7203        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7204            state.enabled = true;
7205            state.state = ModuleState::Restarting;
7206            clear_current_process_facts(state);
7207        })?;
7208        release_dead_registration(
7209            registry,
7210            runtime.forwarding.as_deref(),
7211            snapshot,
7212            &spec.module_id,
7213        )
7214        .await?;
7215    }
7216
7217    reset_restart_count(snapshot, &spec.module_id)?;
7218    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7219    schedule_respawn(
7220        runtime,
7221        snapshot,
7222        &spec.module_id,
7223        runtime.restart_policy.backoff,
7224        RespawnKind::Spawn,
7225    )
7226}
7227
7228async fn reload_child(
7229    spec: &ModuleSpec,
7230    runtime: &SupervisorRuntimeConfig,
7231    registry: &Registry,
7232    process_liveness: &SupervisorProcessLiveness,
7233    snapshot: &SharedSnapshot,
7234    child: &mut Option<SupervisedChild>,
7235) -> Result<(), SuperviseError> {
7236    // Reload cycles a running module; it must not silently start a disabled one.
7237    if !lock_snapshot(snapshot)?.enabled {
7238        return Err(SuperviseError::Disabled {
7239            module_id: spec.module_id.clone(),
7240        });
7241    }
7242    let stop_notice = begin_forwarding_drain(
7243        spec,
7244        runtime,
7245        registry,
7246        snapshot,
7247        Some(true),
7248        RouteCloseReason::Reload,
7249    )
7250    .await?;
7251
7252    if child.is_some() {
7253        drain_optional_child(
7254            &spec.module_id,
7255            spec.protocol,
7256            stop_notice,
7257            registry,
7258            runtime.forwarding.as_deref(),
7259            snapshot,
7260            &runtime.terminal_ring,
7261            &runtime.spawn_events,
7262            child,
7263            runtime.drain_timeout,
7264            ModuleState::Restarting,
7265            Some(true),
7266        )
7267        .await?;
7268    } else {
7269        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7270            state.enabled = true;
7271            state.state = ModuleState::Restarting;
7272            clear_current_process_facts(state);
7273        })?;
7274        release_dead_registration(
7275            registry,
7276            runtime.forwarding.as_deref(),
7277            snapshot,
7278            &spec.module_id,
7279        )
7280        .await?;
7281    }
7282
7283    reset_restart_count(snapshot, &spec.module_id)?;
7284    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7285    schedule_respawn(
7286        runtime,
7287        snapshot,
7288        &spec.module_id,
7289        runtime.restart_policy.backoff,
7290        RespawnKind::Reload,
7291    )
7292}
7293
7294async fn finish_reload_child(
7295    spec: &ModuleSpec,
7296    runtime: &SupervisorRuntimeConfig,
7297    registry: &Registry,
7298    process_liveness: &SupervisorProcessLiveness,
7299    snapshot: &SharedSnapshot,
7300    child: &mut Option<SupervisedChild>,
7301) -> Result<(), SuperviseError> {
7302    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7303    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7304        Ok(next_child) => next_child,
7305        Err(err) => {
7306            return handle_reload_spawn_failure(
7307                spec,
7308                runtime,
7309                process_liveness,
7310                snapshot,
7311                child,
7312                format!("new child failed to spawn: {err}"),
7313            )
7314            .await;
7315        }
7316    };
7317    *child = Some(next_child);
7318
7319    let wait_outcome = {
7320        let active_child = child.as_mut().expect("new reload child was just stored");
7321        wait_for_registration_after_reload(
7322            registry,
7323            &spec.module_id,
7324            snapshot,
7325            active_child,
7326            REGISTRY_RELEASE_TIMEOUT,
7327        )
7328        .await?
7329    };
7330
7331    match wait_outcome {
7332        RegistrationWaitOutcome::Registered => {
7333            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7334            Ok(())
7335        }
7336        RegistrationWaitOutcome::Exited(exit_report) => {
7337            if let Some(active_child) = child.as_mut() {
7338                active_child.drain_stderr(&spec.module_id).await;
7339            }
7340            // Keep the reaped child's roster guard until its terminal is written.
7341            // Shutdown waits on that guard, not on the child Option used for respawn.
7342            let mut exited_child = child.take().expect("exited reload child is still stored");
7343            #[cfg(test)]
7344            if let Some(gate) = &runtime.test_reload_exit_record_gate {
7345                gate.reached.notify_one();
7346                gate.resume.notified().await;
7347            }
7348            let result = handle_reload_child_registration_failure(
7349                spec,
7350                runtime,
7351                registry,
7352                process_liveness,
7353                snapshot,
7354                child,
7355                ReloadRegistrationFailure {
7356                    exit_report: registration_failure_exit_report(exit_report),
7357                    reason: exited_child
7358                        .spawn_failure
7359                        .clone()
7360                        .unwrap_or_else(|| "new child exited before registering".to_string()),
7361                },
7362            )
7363            .await;
7364            exited_child.release_roster();
7365            result
7366        }
7367        RegistrationWaitOutcome::TimedOut => {
7368            let mut timed_out_child = child
7369                .take()
7370                .expect("timed-out reload child is still running");
7371            timed_out_child
7372                .start_kill()
7373                .map_err(|source| SuperviseError::Kill {
7374                    module_id: spec.module_id.clone(),
7375                    source,
7376                })?;
7377            let status = timed_out_child
7378                .wait()
7379                .await
7380                .map_err(|source| SuperviseError::Wait {
7381                    module_id: spec.module_id.clone(),
7382                    source,
7383                })?;
7384            timed_out_child.drain_stderr(&spec.module_id).await;
7385            handle_reload_child_registration_failure(
7386                spec,
7387                runtime,
7388                registry,
7389                process_liveness,
7390                snapshot,
7391                child,
7392                ReloadRegistrationFailure {
7393                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7394                        snapshot,
7395                        &timed_out_child,
7396                        &status,
7397                    )),
7398                    reason: format!(
7399                        "new child did not register within {:?}",
7400                        REGISTRY_RELEASE_TIMEOUT
7401                    ),
7402                },
7403            )
7404            .await
7405        }
7406    }
7407}
7408
7409async fn set_child_enabled(
7410    spec: &ModuleSpec,
7411    runtime: &SupervisorRuntimeConfig,
7412    registry: &Registry,
7413    process_liveness: &SupervisorProcessLiveness,
7414    snapshot: &SharedSnapshot,
7415    child: &mut Option<SupervisedChild>,
7416    enabled: bool,
7417) -> Result<bool, SuperviseError> {
7418    let (current_enabled, current_state, respawn_pending) = {
7419        let state = lock_snapshot(snapshot)?;
7420        (state.enabled, state.state, state.respawn_pending)
7421    };
7422    // `start` (enable on an already-enabled module) heals TERMINAL states instead
7423    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
7424    // clean (Stopped) has no live process and no other in-band recovery — the
7425    // operator's start is the explicit recovery act and resets the budget. Without
7426    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
7427    // which the 2026-07-14 aft outage proved is a trap when the failed module is
7428    // the one providing every agent's shell.
7429    let revive_terminal = enabled
7430        && current_enabled
7431        && child.is_none()
7432        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7433            || (current_state == ModuleState::Restarting && !respawn_pending));
7434    if current_enabled == enabled && !revive_terminal {
7435        return Ok(false);
7436    }
7437
7438    if enabled {
7439        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7440            state.enabled = true;
7441            state.state = ModuleState::Starting;
7442            clear_current_process_facts(state);
7443        })?;
7444        #[cfg(test)]
7445        if runtime.test_seed_stale_facts_before_enable_spawn {
7446            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7447                state.process_alive = true;
7448                state.pid = Some(41);
7449                state.spawned_at_ms = Some(42);
7450                state.spawned_from = Some(PathBuf::from("/spawned/module"));
7451                state.spawned_file_identity = Some(SpawnedFileIdentity {
7452                    device: 43,
7453                    inode: 44,
7454                    #[cfg(windows)]
7455                    windows_image: None,
7456                });
7457            })?;
7458        }
7459        release_dead_registration(
7460            registry,
7461            runtime.forwarding.as_deref(),
7462            snapshot,
7463            &spec.module_id,
7464        )
7465        .await?;
7466        reset_restart_count(snapshot, &spec.module_id)?;
7467        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7468        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7469            Ok(next_child) => next_child,
7470            Err(err) => {
7471                if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7472                    state.state = ModuleState::Failed;
7473                    clear_current_process_facts(state);
7474                }) {
7475                    error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7476                }
7477                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7478                return Err(err);
7479            }
7480        };
7481        *child = Some(next_child);
7482        debug!(module_id = %spec.module_id, "supervised module enabled");
7483        Ok(true)
7484    } else {
7485        let stop_notice = begin_forwarding_drain_if_configured(
7486            spec,
7487            runtime,
7488            registry,
7489            snapshot,
7490            Some(false),
7491            RouteCloseReason::Disable,
7492        )
7493        .await?;
7494        drain_optional_child(
7495            &spec.module_id,
7496            spec.protocol,
7497            stop_notice,
7498            registry,
7499            runtime.forwarding.as_deref(),
7500            snapshot,
7501            &runtime.terminal_ring,
7502            &runtime.spawn_events,
7503            child,
7504            runtime.drain_timeout,
7505            ModuleState::Disabled,
7506            Some(false),
7507        )
7508        .await?;
7509        debug!(module_id = %spec.module_id, "supervised module disabled");
7510        Ok(true)
7511    }
7512}
7513
7514#[allow(clippy::too_many_arguments)]
7515async fn on_child_exit(
7516    spec: &ModuleSpec,
7517    policy: RestartPolicy,
7518    registry: &Registry,
7519    snapshot: &SharedSnapshot,
7520    terminal_ring: &Arc<Mutex<TerminalRing>>,
7521    spawn_events: &SpawnEventFeed,
7522    roster: &ChildRoster,
7523    exit_report: ExitReport,
7524) -> NextAction {
7525    // Once the daemon has begun shutting down, no exit is a crash to recover
7526    // from: the module is exiting because the daemon is going away (EOF on its
7527    // connection, or a service manager signalling the whole cgroup). Record it
7528    // as such and never schedule a respawn, which would only start a process
7529    // for the shutdown to end again.
7530    if roster.is_closed() {
7531        return on_child_exit_during_daemon_shutdown(
7532            spec,
7533            registry,
7534            snapshot,
7535            terminal_ring,
7536            spawn_events,
7537            exit_report,
7538        )
7539        .await;
7540    }
7541    // Every stop the supervisor itself asks for (operator stop, disable,
7542    // restart, reload, swap, a health restart, a drain that runs out of budget)
7543    // takes the child out of the supervise loop and reaps it in
7544    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
7545    // that reaches this point was not requested by the daemon.
7546    //
7547    // For a subc-wire module a clean exit is still a stop: those modules are
7548    // written to re-raise SIGTERM, so a stray outside signal already reads as a
7549    // crash, and exiting 0 is a deliberate choice the module made. A
7550    // `protocol: "none"` module is a stock program we cannot change, and many
7551    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
7552    // stop would leave the module down for good after any stray signal, so it
7553    // goes through the crash path instead: it spends restart budget, respawns
7554    // with the crash backoff, and ends `failed` when the budget runs out.
7555    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7556        && running_protocol(spec, snapshot) == ModuleProtocol::None;
7557    match exit_report.kind {
7558        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7559            info!(
7560                module_id = %spec.module_id,
7561                exit_code = ?exit_report.code,
7562                exit_signal = ?exit_report.signal,
7563                "supervised module exited cleanly"
7564            );
7565            if let Err(err) = publish_after_terminal_record(
7566                &spec.module_id,
7567                || {
7568                    record_terminal(
7569                        &spec.module_id,
7570                        terminal_ring,
7571                        spawn_events,
7572                        &exit_report,
7573                        TerminalDisposition::Stopped,
7574                    )
7575                },
7576                || {
7577                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7578                        state.state = ModuleState::Stopped;
7579                        clear_current_process_facts(state);
7580                        state.last_exit = Some(exit_report.clone());
7581                    })
7582                },
7583            ) {
7584                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7585            }
7586            let registration_released = match wait_for_registration_release(
7587                registry,
7588                &spec.module_id,
7589                REGISTRY_RELEASE_TIMEOUT,
7590            )
7591            .await
7592            {
7593                Ok(()) => true,
7594                Err(err) => {
7595                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7596                    false
7597                }
7598            };
7599            NextAction::Stop {
7600                registration_released,
7601            }
7602        }
7603        ExitKind::Clean | ExitKind::Crash => {
7604            if unrequested_clean_exit_of_protocol_none {
7605                warn!(
7606                    module_id = %spec.module_id,
7607                    exit_code = ?exit_report.code,
7608                    exit_signal = ?exit_report.signal,
7609                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
7610                );
7611            } else {
7612                warn!(
7613                    module_id = %spec.module_id,
7614                    exit_code = ?exit_report.code,
7615                    exit_signal = ?exit_report.signal,
7616                    "supervised module exited abnormally (crash)"
7617                );
7618            }
7619            let mut restart_schedule = None;
7620            let mut disposition = TerminalDisposition::Disabled;
7621            let mut final_state = ModuleState::Disabled;
7622            // Set only when the budget is what stopped the module, so the
7623            // terminal record says which limit was hit rather than leaving
7624            // `failed` to be read as "crashed once, badly".
7625            let mut disposition_detail = lock_snapshot(snapshot)
7626                .ok()
7627                .and_then(|mut state| state.spawn_failure.take());
7628            let now = Instant::now();
7629            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7630                if state.enabled {
7631                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
7632                        // Reserving the last slot exhausts the budget for the
7633                        // replacement, not for this exit. Publish its scheduled
7634                        // recovery atomically with the reservation so connection
7635                        // cleanup sees it while terminal recording still holds
7636                        // the visible state at Running.
7637                        state.respawn_pending = true;
7638                        final_state = ModuleState::Restarting;
7639                        restart_schedule = Some(schedule);
7640                        disposition = TerminalDisposition::Restarting;
7641                    } else {
7642                        final_state = ModuleState::Failed;
7643                        disposition = TerminalDisposition::Failed;
7644                        let budget = policy.budget_exhausted_detail();
7645                        disposition_detail =
7646                            Some(disposition_detail.take().map_or_else(
7647                                || budget.clone(),
7648                                |cause| format!("{cause}; {budget}"),
7649                            ));
7650                    }
7651                } else {
7652                    disposition = TerminalDisposition::Disabled;
7653                }
7654            }) {
7655                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7656                return NextAction::Stop {
7657                    registration_released: false,
7658                };
7659            }
7660            if disposition == TerminalDisposition::Failed {
7661                // The window is in the message, not only in the fields: this line
7662                // is read in a scrollback where a bare `max_restarts=3` reads as a
7663                // lifetime cap and sends the operator looking for three crashes
7664                // that never happened together.
7665                error!(
7666                    module_id = %spec.module_id,
7667                    max_restarts = policy.max_restarts,
7668                    window_secs = policy.window.as_secs(),
7669                    "module stopped: {}",
7670                    policy.budget_exhausted_detail()
7671                );
7672            }
7673            if let Err(err) = publish_after_terminal_record(
7674                &spec.module_id,
7675                || {
7676                    record_terminal_with_detail(
7677                        &spec.module_id,
7678                        terminal_ring,
7679                        spawn_events,
7680                        &exit_report,
7681                        disposition,
7682                        disposition_detail,
7683                    )
7684                },
7685                || {
7686                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7687                        state.state = final_state;
7688                        clear_current_process_facts(state);
7689                        state.last_exit = Some(exit_report.clone());
7690                    })
7691                },
7692            ) {
7693                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7694            }
7695
7696            if let Some(schedule) = restart_schedule {
7697                NextAction::Restart {
7698                    schedule: Some(schedule),
7699                }
7700            } else {
7701                let registration_released = match wait_for_registration_release(
7702                    registry,
7703                    &spec.module_id,
7704                    REGISTRY_RELEASE_TIMEOUT,
7705                )
7706                .await
7707                {
7708                    Ok(()) => true,
7709                    Err(err) => {
7710                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7711                        false
7712                    }
7713                };
7714                NextAction::Stop {
7715                    registration_released,
7716                }
7717            }
7718        }
7719        ExitKind::DeliberateSeverance => {
7720            warn!(
7721                module_id = %spec.module_id,
7722                exit_code = ?exit_report.code,
7723                exit_signal = ?exit_report.signal,
7724                "supervised module exited after deliberate connection severance"
7725            );
7726            let should_restart = match lock_snapshot(snapshot) {
7727                Ok(state) => state.enabled,
7728                Err(err) => {
7729                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
7730                    return NextAction::Stop {
7731                        registration_released: false,
7732                    };
7733                }
7734            };
7735            let final_state = if should_restart {
7736                ModuleState::Restarting
7737            } else {
7738                ModuleState::Disabled
7739            };
7740            if let Err(err) = publish_after_terminal_record(
7741                &spec.module_id,
7742                || {
7743                    record_terminal(
7744                        &spec.module_id,
7745                        terminal_ring,
7746                        spawn_events,
7747                        &exit_report,
7748                        terminal_disposition(final_state),
7749                    )
7750                },
7751                || {
7752                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7753                        state.state = final_state;
7754                        clear_current_process_facts(state);
7755                        state.last_exit = Some(exit_report.clone());
7756                        state.lifetime_restarts += 1;
7757                    })
7758                },
7759            ) {
7760                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
7761                return NextAction::Stop {
7762                    registration_released: false,
7763                };
7764            }
7765
7766            if should_restart {
7767                NextAction::Restart { schedule: None }
7768            } else {
7769                let registration_released = match wait_for_registration_release(
7770                    registry,
7771                    &spec.module_id,
7772                    REGISTRY_RELEASE_TIMEOUT,
7773                )
7774                .await
7775                {
7776                    Ok(()) => true,
7777                    Err(err) => {
7778                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
7779                        false
7780                    }
7781                };
7782                NextAction::Stop {
7783                    registration_released,
7784                }
7785            }
7786        }
7787    }
7788}
7789
7790async fn on_child_exit_during_daemon_shutdown(
7791    spec: &ModuleSpec,
7792    registry: &Registry,
7793    snapshot: &SharedSnapshot,
7794    terminal_ring: &Arc<Mutex<TerminalRing>>,
7795    spawn_events: &SpawnEventFeed,
7796    exit_report: ExitReport,
7797) -> NextAction {
7798    info!(
7799        module_id = %spec.module_id,
7800        exit_code = ?exit_report.code,
7801        exit_signal = ?exit_report.signal,
7802        exit_kind = ?exit_report.kind,
7803        "supervised module exited during daemon shutdown; not restarting it"
7804    );
7805    if let Err(err) = publish_after_terminal_record(
7806        &spec.module_id,
7807        || {
7808            record_terminal(
7809                &spec.module_id,
7810                terminal_ring,
7811                spawn_events,
7812                &exit_report,
7813                TerminalDisposition::DaemonShutdown,
7814            )
7815        },
7816        || {
7817            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7818                state.state = ModuleState::Stopped;
7819                clear_current_process_facts(state);
7820                state.last_exit = Some(exit_report.clone());
7821            })
7822        },
7823    ) {
7824        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
7825    }
7826    let registration_released =
7827        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
7828            .await
7829            .is_ok();
7830    NextAction::Stop {
7831        registration_released,
7832    }
7833}
7834
7835// Keep the status lock out of terminal I/O. Readers may observe the previous
7836// state while recording is in progress, but never a completed exit without its
7837// history. Journal errors are logged by the journal and retained in the ring;
7838// a recording panic still publishes the final state before resuming the unwind.
7839fn publish_after_terminal_record<T>(
7840    module_id: &str,
7841    record: impl FnOnce(),
7842    publish: impl FnOnce() -> T,
7843) -> T {
7844    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
7845    let published = publish();
7846    if let Err(panic) = recorded {
7847        error!(
7848            module_id,
7849            "terminal recording panicked; published final state before resuming unwind"
7850        );
7851        std::panic::resume_unwind(panic);
7852    }
7853    published
7854}
7855
7856fn fail_after_wait_error(
7857    module_id: &str,
7858    snapshot: &SharedSnapshot,
7859    terminal_ring: &Arc<Mutex<TerminalRing>>,
7860    spawn_events: &SpawnEventFeed,
7861    detail: String,
7862) {
7863    let report = wait_error_exit_report();
7864    publish_after_terminal_record(
7865        module_id,
7866        || {
7867            record_terminal_with_detail(
7868                module_id,
7869                terminal_ring,
7870                spawn_events,
7871                &report,
7872                TerminalDisposition::Failed,
7873                Some(detail),
7874            )
7875        },
7876        || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
7877    );
7878}
7879
7880#[cfg(test)]
7881fn record_wait_error_terminal(
7882    module_id: &str,
7883    terminal_ring: &Arc<Mutex<TerminalRing>>,
7884    spawn_events: &SpawnEventFeed,
7885) {
7886    record_terminal(
7887        module_id,
7888        terminal_ring,
7889        spawn_events,
7890        &wait_error_exit_report(),
7891        TerminalDisposition::Failed,
7892    );
7893}
7894
7895fn record_terminal(
7896    module_id: &str,
7897    terminal_ring: &Arc<Mutex<TerminalRing>>,
7898    spawn_events: &SpawnEventFeed,
7899    exit_report: &ExitReport,
7900    disposition: TerminalDisposition,
7901) {
7902    record_terminal_with_detail(
7903        module_id,
7904        terminal_ring,
7905        spawn_events,
7906        exit_report,
7907        disposition,
7908        None,
7909    );
7910}
7911
7912/// The ring lock is held only to capture the read (see
7913/// `TerminalJournal::capture_read`), so this module's exits keep recording
7914/// while the journal files are read. Blocking: it reads files.
7915fn durable_terminal_history_of(
7916    terminal_ring: &Mutex<TerminalRing>,
7917    module_id: &str,
7918) -> subc_control::TerminalHistory {
7919    let read = terminal_ring
7920        .lock()
7921        .unwrap_or_else(|p| p.into_inner())
7922        .capture_durable_history();
7923    read.read(module_id)
7924}
7925
7926fn record_terminal_with_detail(
7927    module_id: &str,
7928    terminal_ring: &Arc<Mutex<TerminalRing>>,
7929    spawn_events: &SpawnEventFeed,
7930    exit_report: &ExitReport,
7931    disposition: TerminalDisposition,
7932    disposition_detail: Option<String>,
7933) {
7934    #[cfg(test)]
7935    terminal_history_tests::before_terminal_record(spawn_events);
7936    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
7937    let record = TerminalRecord {
7938        exit_code: exit_report.code,
7939        exit_signal: exit_report.signal,
7940        at_ms: exit_report.at_ms,
7941        disposition,
7942        exit_kind: exit_report.kind.into(),
7943        disposition_detail,
7944    };
7945    terminal_ring
7946        .lock()
7947        .unwrap_or_else(|poisoned| poisoned.into_inner())
7948        .record_exit(module_id, record);
7949}
7950
7951fn untrack_if_registration_released(
7952    process_liveness: &SupervisorProcessLiveness,
7953    registry: &Registry,
7954    module_id: &str,
7955    snapshot: &SharedSnapshot,
7956) {
7957    match registry.get_module(module_id) {
7958        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
7959        Ok(Some(_)) => {}
7960        Err(err) => {
7961            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
7962        }
7963    }
7964}
7965
7966/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
7967/// then apply the module's configured entries minus daemon-private capture keys.
7968///
7969/// Separated from `spawn_child` only so it can be asserted without spawning a
7970/// process — a duplicate of this logic in a test would pass while the real one
7971/// drifted, which is the defect class this function exists to avoid.
7972/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
7973/// nonce. A `protocol: "none"` module gets neither, because it cannot use
7974/// either and the argument would stop a stock binary from starting at all.
7975/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
7976///
7977/// The plain-spawn form, kept for the tests that assert its plan; spawns go
7978/// through [`apply_wire_spawn_args_for_role`].
7979#[cfg(test)]
7980fn apply_wire_spawn_args(
7981    command: &mut Command,
7982    spec: &ModuleSpec,
7983    connection_file_path: Option<&std::path::Path>,
7984    handle: Option<&SupervisorHandle>,
7985) -> Result<Option<NonceHandoff>, SuperviseError> {
7986    apply_wire_spawn_args_for_role(
7987        command,
7988        spec,
7989        connection_file_path,
7990        handle,
7991        SpawnRole::Plain,
7992    )
7993}
7994
7995/// The read end of a spawn's launch-nonce pipe, prepared by
7996/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
7997/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
7998/// server before spawn and authenticates its client after the child exists.
7999#[cfg(unix)]
8000type NonceHandoff = subc_os::LaunchNonceHandoff;
8001#[cfg(windows)]
8002type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8003#[cfg(not(any(unix, windows)))]
8004type NonceHandoff = std::convert::Infallible;
8005
8006/// Create the launch nonce that a new module process must present when it
8007/// registers or opens a consumer connection, including a replacement process
8008/// started alongside a still-serving module.
8009///
8010/// A plain spawn replaces the module's recorded nonce. A swap candidate records
8011/// a separate candidate token so the still-serving incumbent and its consumers
8012/// keep their nonce. Both records are installed before the process exists, so
8013/// the child's initial HELLO registration cannot arrive ahead of its nonce.
8014///
8015/// On Unix the nonce is delivered only through a pipe. It is written into
8016/// a pipe whose read end the child gets as descriptor 3, named by
8017/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
8018/// process of the same user cannot read it with `ps eww`. That handoff is
8019/// returned rather than installed here, because installing it replaces
8020/// whatever the child has at descriptor 3 and so must be the last pre-exec
8021/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
8022/// the Linux callback that writes the process into its cgroup. Only after that
8023/// does `handoff.install_last` register the descriptor-3 replacement, so it
8024/// cannot close a descriptor the cgroup callback still needs.
8025/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
8026fn apply_wire_spawn_args_for_role(
8027    command: &mut Command,
8028    spec: &ModuleSpec,
8029    connection_file_path: Option<&std::path::Path>,
8030    handle: Option<&SupervisorHandle>,
8031    role: SpawnRole,
8032) -> Result<Option<NonceHandoff>, SuperviseError> {
8033    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8034    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
8035    // included: a daemon started from a module's process tree inherits it,
8036    // and passing it on would point the child at a descriptor it does not
8037    // have.
8038    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8039    #[cfg(windows)]
8040    {
8041        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8042        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8043    }
8044    // Remove inherited or configured copies too: withholding must mean absent.
8045    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8046    if spec.protocol == ModuleProtocol::None {
8047        return Ok(None);
8048    }
8049    if let Some(connection_file_path) = connection_file_path {
8050        command.arg(SUBC_ARG).arg(connection_file_path);
8051    }
8052
8053    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
8054    // route.open attestation. Reserved modules additionally use the same nonce
8055    // for HELLO id-squatting protection. A respawn rotates both records.
8056    let nonce = generate_launch_nonce()?;
8057    if let Some(handle) = handle {
8058        match role {
8059            SpawnRole::Plain => {
8060                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8061                if spec.reserved {
8062                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8063                }
8064            }
8065            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8066        }
8067    }
8068    #[cfg(unix)]
8069    let handoff = {
8070        let handoff =
8071            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8072                program: spec.program.clone(),
8073                source,
8074                cgroup_path: None,
8075            })?;
8076        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8077        Some(handoff)
8078    };
8079    #[cfg(windows)]
8080    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8081    #[cfg(not(any(unix, windows)))]
8082    let handoff = None;
8083    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
8084    // until the daemon's per-spawn delivery records show that every running
8085    // Windows module has read its secret from the named pipe.
8086    #[cfg(not(unix))]
8087    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8088    Ok(handoff)
8089}
8090
8091fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8092    command.env_remove(CK_LOG_ENV);
8093    // The spawn role is the supervisor's to set, and only on a swap candidate
8094    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8095    // it, is what makes it absent on a plain spawn: the daemon's own
8096    // environment could carry it, and so could a spec built outside daemon
8097    // config (config refuses it as an `env` key). A module reading it on a
8098    // plain restart would pick the long swap budget and leave callers waiting.
8099    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8100    for (key, value) in &spec.env {
8101        // cortexkit-log currently exposes retention only as a Rust struct, not
8102        // environment names. These values are daemon-private sink metadata and
8103        // must never become a public child-process contract by being inherited.
8104        if matches!(
8105            key.as_str(),
8106            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8107        ) || key == SUBC_SPAWN_ROLE_ENV
8108        {
8109            continue;
8110        }
8111        command.env(key, value);
8112    }
8113}
8114
8115#[cfg(windows)]
8116fn prepare_windows_nonce_pipe(
8117    command: &mut Command,
8118    spec: &ModuleSpec,
8119    nonce: &str,
8120) -> Option<NonceHandoff> {
8121    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8122        Ok(handoff) => {
8123            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8124            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8125            // still register until every module reads the pipe. A reader without
8126            // this variable set to env returns the pipe error instead of using
8127            // an inherited or configured environment copy.
8128            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8129            Some(handoff)
8130        }
8131        Err(error) => {
8132            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8133            None
8134        }
8135    }
8136}
8137
8138#[cfg(windows)]
8139fn start_windows_nonce_pipe(
8140    handoff: Option<NonceHandoff>,
8141    pid: u32,
8142    spec: &ModuleSpec,
8143) -> Option<subc_os::LaunchNoncePipeDelivery> {
8144    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8145    // same maximum wait used for registration during a reload. Older modules
8146    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8147    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8148        Ok(delivery) => Some(delivery),
8149        Err(error) => {
8150            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8151            None
8152        }
8153    })
8154}
8155
8156fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8157    #[cfg(windows)]
8158    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8159        return Some(
8160            if snapshot
8161                .pipe_consumed
8162                .as_ref()
8163                .is_some_and(|consumed| consumed.get().is_some())
8164            {
8165                "pipe"
8166            } else {
8167                "env"
8168            }
8169            .to_owned(),
8170        );
8171    }
8172    let _ = snapshot;
8173    None
8174}
8175
8176/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8177/// of a blue/green swap.
8178#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8179enum SpawnRole {
8180    Plain,
8181    SwapCandidate,
8182}
8183
8184/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8185/// `apply_child_env` has already removed the variable for every spawn.
8186fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8187    if role == SpawnRole::SwapCandidate {
8188        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8189    }
8190}
8191
8192fn previous_instance_error(
8193    spec: &ModuleSpec,
8194    pid: u32,
8195    reason: crate::live_children::RetainedReason,
8196) -> SuperviseError {
8197    SuperviseError::Spawn {
8198        program: spec.program.clone(),
8199        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8200        cgroup_path: None,
8201    }
8202}
8203
8204fn spawn_child(
8205    spec: &ModuleSpec,
8206    connection_file_path: Option<&std::path::Path>,
8207    handle: Option<&SupervisorHandle>,
8208    ring: &Arc<Mutex<StderrRing>>,
8209    capture_logs_dir: Option<&std::path::Path>,
8210    roster: &ChildRoster,
8211    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8212) -> Result<SupervisedChild, SuperviseError> {
8213    spawn_child_in_slot(
8214        spec,
8215        connection_file_path,
8216        handle,
8217        ring,
8218        capture_logs_dir,
8219        roster,
8220        #[cfg(target_os = "linux")]
8221        cgroup_placement,
8222        SpawnRole::Plain,
8223        false,
8224    )
8225}
8226
8227/// Spawn one process of `spec` into a slot.
8228///
8229/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
8230/// A swap candidate needs a different cgroup from the process it is replacing,
8231/// which is still alive: in the same cgroup the two would be one kill domain,
8232/// and killing a failed candidate could take the incumbent with it.
8233///
8234/// The stderr capture file is `<module_id>.stderr.log` for every process of
8235/// the module, whichever slot it is in, because that is the one file
8236/// `ck module logs` reads. During a swap's overlap both processes append to it;
8237/// the daemon writes whole lines, so the two interleave by line, which is also
8238/// the merged view an operator wants while a swap runs.
8239#[allow(clippy::too_many_arguments)]
8240fn spawn_child_in_slot(
8241    spec: &ModuleSpec,
8242    connection_file_path: Option<&std::path::Path>,
8243    handle: Option<&SupervisorHandle>,
8244    ring: &Arc<Mutex<StderrRing>>,
8245    capture_logs_dir: Option<&std::path::Path>,
8246    roster: &ChildRoster,
8247    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8248    role: SpawnRole,
8249    alternate_slot: bool,
8250) -> Result<SupervisedChild, SuperviseError> {
8251    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8252        return Err(previous_instance_error(spec, pid, reason));
8253    }
8254    if roster.is_closed() {
8255        return Err(SuperviseError::Spawn {
8256            program: spec.program.clone(),
8257            source: io::Error::other("the daemon is shutting down; not starting a new process"),
8258            cgroup_path: None,
8259        });
8260    }
8261    #[cfg(target_os = "linux")]
8262    let cgroup_name = {
8263        // Slot names alone are not kill domains: a retired incumbent may still
8264        // be draining when a later enable/restart spawns into the same slot.
8265        // Decimal entropy keeps the suffix unambiguous; Placement performs
8266        // the module-id escaping and constructs the filesystem path.
8267        if cgroup_placement.is_none() {
8268            swap::cgroup_name(&spec.module_id, alternate_slot)
8269        } else {
8270            let nonce = generate_launch_nonce()?;
8271            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8272            // Leave room for byte escaping and the suffix under NAME_MAX. The
8273            // label is only for humans; the nonce identifies the kill domain.
8274            let mut end = spec.module_id.len().min(64);
8275            while !spec.module_id.is_char_boundary(end) {
8276                end -= 1;
8277            }
8278            format!(
8279                "{}_{suffix}",
8280                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8281            )
8282        }
8283    };
8284    #[cfg(not(target_os = "linux"))]
8285    let _ = alternate_slot;
8286    #[cfg(target_os = "macos")]
8287    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8288    #[cfg(windows)]
8289    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8290        program: spec.program.clone(),
8291        source,
8292        cgroup_path: None,
8293    })?;
8294    #[cfg(windows)]
8295    let mut command = Command::new(image_capture.path());
8296    #[cfg(not(any(target_os = "macos", windows)))]
8297    let mut command = Command::new(&spec.program);
8298    command.args(&spec.args);
8299    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
8300    // that is the whole of the intent, so remove that one key rather than the
8301    // environment.
8302    //
8303    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
8304    // and took the POSIX environment with it. Modules spawned that way had no
8305    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
8306    // logging:
8307    //
8308    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
8309    //     both unset it fell back to the temp dir alone and `ck` could not find
8310    //     a daemon running on the same machine from inside any module's process
8311    //     tree — reporting a path the file has never lived at, which reads as
8312    //     "the daemon did not write its file".
8313    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
8314    //     the RELATIVE `.local/share`, so a module deriving its own store path
8315    //     resolved it against its own CWD. That is the store-fragmentation
8316    //     defect the daemon already refuses in config (`parse_doc` rejects a
8317    //     relative `storage.data_home`) arriving by derivation instead.
8318    //   * anything a module spawns inherited it: git without ~/.gitconfig,
8319    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
8320    //     quietly rather than erroring.
8321    //
8322    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
8323    // offered one candidate under /tmp while the file sat in /run/user/1000.
8324    //
8325    // A configured module is unaffected either way: `module_spec()` puts the
8326    // resolved CK_LOG into `spec.env`, which is applied below and therefore
8327    // wins over anything ambient.
8328    apply_child_env(&mut command, spec);
8329    apply_spawn_role(&mut command, role);
8330    let nonce_handoff =
8331        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8332
8333    #[cfg(target_os = "linux")]
8334    let cgroup_path = cgroup_placement
8335        .map(|placement| placement.module_path(&cgroup_name))
8336        .transpose()
8337        .map_err(|source| SuperviseError::Cgroup {
8338            module_id: spec.module_id.clone(),
8339            source,
8340        })?;
8341    #[cfg(not(target_os = "linux"))]
8342    let cgroup_path: Option<PathBuf> = None;
8343    #[cfg(target_os = "linux")]
8344    if let Some(path) = &cgroup_path {
8345        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8346            if let Some(placement) = cgroup_placement {
8347                remove_module_cgroup(placement, &cgroup_name);
8348            }
8349            return Err(error);
8350        }
8351    }
8352
8353    let output_sink = if let Some(logs_dir) = capture_logs_dir {
8354        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8355        match ChildOutputSink::open(&path, capture_retention(spec)) {
8356            Ok(sink) => sink,
8357            Err(error) => {
8358                warn!(
8359                    module_id = %spec.module_id,
8360                    path = %path.display(),
8361                    error = %error,
8362                    "could not open child output capture file; forwarding to stderr"
8363                );
8364                ChildOutputSink::Stderr
8365            }
8366        }
8367    } else {
8368        ChildOutputSink::Stderr
8369    };
8370
8371    command.stdout(Stdio::piped());
8372    command.stderr(Stdio::piped());
8373    command.kill_on_drop(true);
8374    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
8375    // before exec). In the daemon's group, a service manager that kills the
8376    // job's process group when the daemon exits (launchd's default) killed
8377    // every module at the same moment its control connection closed, so no
8378    // module ever ran its EOF teardown on a daemon stop. Outside that group a
8379    // module is reached only by the daemon: the EOF it sees when its
8380    // connection closes, and the bounded stop in `child_roster` for anything
8381    // still running after that. On Linux this composes with the cgroup
8382    // placement above: that is a pre_exec write to cgroup.procs, std performs
8383    // setpgid in the child before running pre_exec callbacks, and the two
8384    // change independent process attributes.
8385    //
8386    // stdin is /dev/null because a process outside the terminal's foreground
8387    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
8388    // by hand would otherwise hand down. Under a service manager stdin is
8389    // already /dev/null.
8390    #[cfg(unix)]
8391    command.process_group(0);
8392    command.stdin(Stdio::null());
8393    // The LAST pre-exec step, after the cgroup placement above: installing the
8394    // nonce at descriptor 3 replaces whatever the child had there, which could
8395    // be the descriptor an earlier step writes through.
8396    #[cfg(unix)]
8397    if let Some(handoff) = nonce_handoff {
8398        handoff.install_last(command.as_std_mut());
8399    }
8400    #[cfg(not(any(unix, windows)))]
8401    let _ = nonce_handoff;
8402
8403    // Containment, step 1 of 3 (issue #109): create the child suspended so it
8404    // cannot run a single instruction -- and therefore cannot spawn a
8405    // grandchild -- before it is in the job. See `contain_spawned_child` for the
8406    // other two steps and why the window matters.
8407    #[cfg(windows)]
8408    {
8409        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8410            program: spec.program.clone(),
8411            source,
8412            cgroup_path: None,
8413        })?;
8414        subc_jobobject::suspend_on_create_async(&mut command);
8415    }
8416    let mut child = match command.spawn() {
8417        Ok(child) => child,
8418        Err(source) => {
8419            #[cfg(target_os = "linux")]
8420            if let Some(placement) = cgroup_placement {
8421                remove_module_cgroup(placement, &cgroup_name);
8422            }
8423            return Err(SuperviseError::Spawn {
8424                program: spec.program.clone(),
8425                source,
8426                cgroup_path,
8427            });
8428        }
8429    };
8430    #[cfg(test)]
8431    spawn_record_tests::after_create(&child, spec);
8432    // The parent must close its writer now: the acknowledgement pipe reports EOF
8433    // only when every writer is gone, and the child's copy closes when the
8434    // trampoline replaces itself with the module. Command holds only an integer
8435    // in its pre_exec callback, not another writer.
8436    #[cfg(target_os = "macos")]
8437    drop(exec_ack);
8438
8439    // Bind while the child is still suspended, before job assignment resumes it.
8440    // Keeping the executable handle open across CreateProcessW denies rename
8441    // and deletion, so the file cannot be exchanged during spawn. Bind that
8442    // file ID to the process handle returned by spawn, never a reopened PID.
8443    #[cfg(windows)]
8444    let windows_image = match image_capture.bind_tokio(&child) {
8445        Ok(image) => Arc::new(image),
8446        Err(source) => {
8447            kill_and_reap_rejected_child(&mut child);
8448            return Err(SuperviseError::Spawn {
8449                program: spec.program.clone(),
8450                source,
8451                cgroup_path: None,
8452            });
8453        }
8454    };
8455    // Assign the suspended child to its module job; resume only after its
8456    // process identity has been durably recorded for orphan recovery.
8457    #[cfg(windows)]
8458    let job = contain_spawned_child(&mut child, spec)?;
8459    let spawned_at_ms = unix_ms_now();
8460    #[cfg(not(windows))]
8461    let spawned_from = spec.program.clone();
8462    #[cfg(windows)]
8463    let spawned_from = windows_image.path().to_path_buf();
8464    #[cfg(not(windows))]
8465    let spawned_file_identity = spawned_file_identity(&spawned_from);
8466    #[cfg(windows)]
8467    let spawned_file_identity = Some(SpawnedFileIdentity {
8468        device: 0,
8469        inode: 0,
8470        windows_image: Some(Arc::clone(&windows_image)),
8471    });
8472    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8473        program: spec.program.clone(),
8474        source: io::Error::other("spawned child exposed no live pid"),
8475        cgroup_path: cgroup_path.clone(),
8476    })?;
8477    #[cfg(windows)]
8478    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8479    #[cfg(not(windows))]
8480    let process_start_time = crate::provenance::process_start_time(pid);
8481    #[cfg(windows)]
8482    let process_start_time = Some(windows_image.start_time());
8483    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8484    #[cfg(all(test, target_os = "macos"))]
8485    privacy_exec_boundary_tests::before_image_sample(spec, pid);
8486    // Record the committed kernel image, not the configured pathname: PATH
8487    // lookup and shebang interpretation can select a different executable.
8488    // Linux's posix_spawn can return before that image is installed, so the
8489    // observer also waits for the kernel's first-exec flag to clear.
8490    #[cfg(not(windows))]
8491    let recorded_image = observe_spawned_image(pid);
8492    // spawn() confirms only the first exec, into the trampoline. Never persist
8493    // the trampoline image; the asynchronous acknowledgement publishes the
8494    // module image once the trampoline has replaced itself with the module.
8495    #[cfg(target_os = "macos")]
8496    let recorded_image = if privacy_exec.is_some() {
8497        None
8498    } else {
8499        recorded_image
8500    };
8501    #[cfg(target_os = "linux")]
8502    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8503    #[cfg(not(target_os = "linux"))]
8504    let recorded_cgroup_name = None;
8505    let roster_guard = roster.admit(
8506        spec.module_id.clone(),
8507        pid,
8508        spec.protocol,
8509        process_start_time,
8510        crate::child_roster::RecordedIdentity {
8511            #[cfg(not(windows))]
8512            start_time: recorded_image.map(|image| image.start_time),
8513            #[cfg(windows)]
8514            start_time: process_start_time,
8515            #[cfg(not(windows))]
8516            executable: recorded_image
8517                .and_then(|image| image.executable)
8518                .map(crate::live_children::ExecutableIdentity::from),
8519            #[cfg(windows)]
8520            executable: Some(crate::live_children::ExecutableIdentity::from(
8521                windows_image.identity(),
8522            )),
8523            cgroup_name: recorded_cgroup_name,
8524            #[cfg(target_os = "linux")]
8525            cgroup_placement: cgroup_placement.cloned(),
8526        },
8527    );
8528    let roster_guard = match roster_guard {
8529        Ok(guard) => guard,
8530        Err(source) => {
8531            #[cfg(windows)]
8532            if let Some(job) = &job {
8533                let _ = job.terminate();
8534            }
8535            #[cfg(target_os = "linux")]
8536            kill_module_cgroup(cgroup_placement, &cgroup_name);
8537            kill_and_reap_rejected_child(&mut child);
8538            #[cfg(target_os = "linux")]
8539            if let Some(placement) = cgroup_placement {
8540                remove_module_cgroup(placement, &cgroup_name);
8541            }
8542            return Err(SuperviseError::Spawn {
8543                program: spec.program.clone(),
8544                source: io::Error::other(format!(
8545                    "could not durably record module child: {source}"
8546                )),
8547                cgroup_path,
8548            });
8549        }
8550    };
8551    // The check at the top of this function can pass just before daemon
8552    // shutdown begins, and the process is only in the roster from here on.
8553    // The shutdown stop returns as soon as it finds the roster empty, so a
8554    // process admitted after that look would outlive the daemon. The roster
8555    // is closed before the stop first reads it and admission happens under
8556    // the roster's lock, so either the stop sees this process or this check
8557    // sees the roster closed: end the process now rather than start a module
8558    // the daemon is about to stop.
8559    if roster.is_closed() {
8560        // This child was never admitted, so there is no module protocol shutdown to wait for.
8561        #[cfg(target_os = "linux")]
8562        kill_module_cgroup(cgroup_placement, &cgroup_name);
8563        if let Err(error) = child.start_kill() {
8564            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8565        }
8566        #[cfg(target_os = "linux")]
8567        if let Some(placement) = cgroup_placement {
8568            // This spawn was never admitted, so shutdown has no roster entry
8569            // to await. Do not detach its cleanup: the runtime could exit
8570            // before that task reaps the rejected child and removes its group.
8571            while matches!(child.try_wait(), Ok(None)) {
8572                std::thread::yield_now();
8573            }
8574            if matches!(
8575                subc_cgroup::kill_module(Some(placement), &cgroup_name),
8576                subc_cgroup::KillOutcome::Killed
8577            ) {
8578                if let Ok(path) = placement.module_path(&cgroup_name) {
8579                    while std::fs::read_to_string(path.join("cgroup.events"))
8580                        .ok()
8581                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8582                    {
8583                        std::thread::yield_now();
8584                    }
8585                }
8586            }
8587            remove_module_cgroup(placement, &cgroup_name);
8588        }
8589        drop(roster_guard);
8590        return Err(SuperviseError::Spawn {
8591            program: spec.program.clone(),
8592            source: io::Error::other(
8593                "the daemon began shutting down while this process was starting; ended it",
8594            ),
8595            cgroup_path,
8596        });
8597    }
8598
8599    #[cfg(windows)]
8600    if let Err(error) = resume_suspended_child(pid, spec) {
8601        kill_and_reap_rejected_child(&mut child);
8602        return Err(error);
8603    }
8604
8605    let stdout_pump = match child.stdout.take() {
8606        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8607        None => {
8608            warn!(
8609                module_id = %spec.module_id,
8610                "spawned child exposed no stdout pipe; file capture will be incomplete"
8611            );
8612            None
8613        }
8614    };
8615    let stderr_pump = match child.stderr.take() {
8616        Some(stderr) => {
8617            let generation = ring
8618                .lock()
8619                .unwrap_or_else(|poisoned| poisoned.into_inner())
8620                .begin_process();
8621            Some(StderrPump {
8622                task: tokio::spawn(pump_stderr_to(
8623                    stderr,
8624                    Arc::clone(ring),
8625                    generation,
8626                    output_sink,
8627                )),
8628                generation,
8629            })
8630        }
8631        None => {
8632            // Spawning succeeded but the pipe did not materialise. Recording it as
8633            // uncaptured keeps the tail honest: the alternative is an empty tail
8634            // that reads as a module which printed nothing.
8635            ring.lock()
8636                .unwrap_or_else(|poisoned| poisoned.into_inner())
8637                .mark_not_captured("stderr pipe was not available on spawn");
8638            warn!(
8639                module_id = %spec.module_id,
8640                "spawned child exposed no stderr pipe; tail will be unavailable"
8641            );
8642            None
8643        }
8644    };
8645
8646    let exec = Arc::new(crate::activity::ExecOutcome::new());
8647    #[cfg(target_os = "macos")]
8648    let pending_exec = privacy_exec.is_some();
8649    #[cfg(not(target_os = "macos"))]
8650    let pending_exec = false;
8651    if !pending_exec {
8652        exec.finish(crate::activity::ExecState::CommittedExec);
8653    }
8654    Ok(SupervisedChild {
8655        child,
8656        exec: crate::activity::ExecOwner(exec),
8657        #[cfg(windows)]
8658        nonce_delivery,
8659        protocol: spec.protocol,
8660        #[cfg(target_os = "linux")]
8661        module_id: cgroup_name,
8662        #[cfg(target_os = "linux")]
8663        cgroup_placement: cgroup_placement.cloned(),
8664        #[cfg(windows)]
8665        job,
8666        stdout_pump,
8667        stderr_pump,
8668        stderr_ring: Arc::clone(ring),
8669        spawned_at_ms,
8670        spawned_from,
8671        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
8672        spawned_file_identity,
8673        process_start_time,
8674        process_identity,
8675        pid,
8676        roster_guard: Some(roster_guard),
8677        #[cfg(target_os = "macos")]
8678        privacy_exec,
8679        #[cfg(target_os = "macos")]
8680        report_ready: Arc::new(OnceLock::new()),
8681        spawn_failure: None,
8682    })
8683}
8684
8685#[cfg(test)]
8686mod spawn_record_tests {
8687    use super::*;
8688
8689    thread_local! {
8690        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8691    }
8692
8693    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8694        LAST_PID.set(child.id());
8695        #[cfg(windows)]
8696        if let Some((_, address)) = _spec
8697            .env
8698            .iter()
8699            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8700        {
8701            use std::io::{Read, Write};
8702            let mut stream = std::net::TcpStream::connect(address).unwrap();
8703            stream
8704                .write_all(&child.id().unwrap().to_ne_bytes())
8705                .unwrap();
8706            let mut release = [0];
8707            stream.read_exact(&mut release).unwrap();
8708        }
8709    }
8710
8711    #[cfg(windows)]
8712    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8713        if spec
8714            .env
8715            .iter()
8716            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8717        {
8718            Err(io::Error::other("injected module job assignment failure"))
8719        } else {
8720            Ok(())
8721        }
8722    }
8723
8724    fn spec() -> ModuleSpec {
8725        ModuleSpec {
8726            module_id: "spawn-record-worker".into(),
8727            #[cfg(unix)]
8728            program: "/bin/sleep".into(),
8729            #[cfg(windows)]
8730            program: std::env::current_exe().unwrap(),
8731            #[cfg(unix)]
8732            args: vec!["60".into()],
8733            #[cfg(windows)]
8734            args: vec![
8735                "--exact".into(),
8736                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
8737                "--nocapture".into(),
8738            ],
8739            env: vec![],
8740            reserved: false,
8741            reserved_prefixes: vec![],
8742            protocol: ModuleProtocol::None,
8743            overlap: Default::default(),
8744        }
8745    }
8746
8747    #[tokio::test]
8748    async fn record_sync_failure_fails_spawn_and_reaps_child() {
8749        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
8750        let supervisor =
8751            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8752                .with_live_children_record(home.join("live-children.json"));
8753        crate::live_children::fail_next_file_sync();
8754        let result = supervisor.spawn(spec());
8755        let failed = result.is_err();
8756        if let Ok(module) = result {
8757            module.stop().await.unwrap();
8758        }
8759        assert!(failed, "record failure must fail spawn");
8760        let pid = LAST_PID.get().expect("the real OS child was created");
8761        assert!(
8762            subc_os::Process::open(pid).unwrap().is_none(),
8763            "child {pid} survived rejected admission or was not reaped"
8764        );
8765    }
8766
8767    #[cfg(target_os = "macos")]
8768    #[test]
8769    #[ignore = "manual on-device latency measurement"]
8770    fn durable_record_write_latency_50() {
8771        let home = cortexkit_test_support::ScratchDir::new("record-latency");
8772        let path = home.join("live-children.json");
8773        let children = vec![crate::live_children::LiveChild {
8774            module_id: "latency-worker".into(),
8775            pid: std::process::id(),
8776            protocol: ModuleProtocol::None,
8777            start_time: Some(1),
8778            executable: None,
8779            cgroup_name: None,
8780        }];
8781        let mut samples = Vec::new();
8782        for _ in 0..50 {
8783            let start = std::time::Instant::now();
8784            crate::live_children::write_record(&path, &children).unwrap();
8785            samples.push(start.elapsed());
8786        }
8787        samples.sort();
8788        eprintln!(
8789            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
8790            (samples[24] + samples[25]) / 2,
8791            samples[47]
8792        );
8793    }
8794
8795    #[cfg(windows)]
8796    #[test]
8797    fn windows_store_touch_fixture() {
8798        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
8799            std::fs::write(path, b"module ran").unwrap();
8800        }
8801    }
8802
8803    #[cfg(windows)]
8804    #[tokio::test]
8805    async fn windows_daemon_fixture() {
8806        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
8807            return;
8808        };
8809        let mut spec = spec();
8810        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
8811        let supervisor =
8812            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8813        let _module = supervisor.spawn(spec).unwrap();
8814    }
8815
8816    #[cfg(windows)]
8817    #[test]
8818    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
8819        use std::io::Read;
8820        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
8821        // An outer job models a scheduler/service host. The daemon's root and
8822        // root job must nest without requiring breakaway from this job.
8823        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8824        command
8825            .args([
8826                "--exact",
8827                "supervise::spawn_record_tests::windows_daemon_fixture",
8828                "--nocapture",
8829            ])
8830            .env(
8831                "SUBC_TEST_DAEMON_BARRIER",
8832                listener.local_addr().unwrap().to_string(),
8833            );
8834        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8835        listener.set_nonblocking(true).unwrap();
8836        let deadline = std::time::Instant::now() + Duration::from_secs(30);
8837        let mut stream = loop {
8838            if let Ok((stream, _)) = listener.accept() {
8839                break stream;
8840            }
8841            assert!(
8842                daemon.child.try_wait().unwrap().is_none(),
8843                "daemon failed before rendezvous"
8844            );
8845            assert!(
8846                std::time::Instant::now() < deadline,
8847                "daemon did not reach CreateProcess failpoint"
8848            );
8849            std::thread::yield_now();
8850        };
8851        stream
8852            .set_read_timeout(Some(Duration::from_secs(30)))
8853            .unwrap();
8854        let mut pid = [0; 4];
8855        stream.read_exact(&mut pid).unwrap();
8856        let pid = u32::from_ne_bytes(pid);
8857        assert!(subc_jobobject::process_exists(pid));
8858        // Terminate only the daemon, not its outer job: otherwise the host job
8859        // would also kill a child whose daemon never joined its root job.
8860        daemon.child.kill().unwrap();
8861        daemon.child.wait().unwrap();
8862        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
8863        if !exited {
8864            daemon.job.terminate().unwrap();
8865        }
8866        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
8867    }
8868
8869    #[cfg(windows)]
8870    #[tokio::test]
8871    async fn windows_containment_failure_never_resumes() {
8872        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
8873        let sentinel = home.join("store-touched");
8874        let mut spec = spec();
8875        spec.env
8876            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
8877        spec.env.push((
8878            "SUBC_TEST_STORE_TOUCH".into(),
8879            sentinel.display().to_string(),
8880        ));
8881        let supervisor =
8882            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8883        let result = supervisor.spawn(spec);
8884        // A version that resumes despite assignment failure must have time to
8885        // execute its sentinel before dropping the returned module kills it.
8886        if let Ok(module) = result {
8887            let _ = tokio::time::timeout(Duration::from_secs(10), async {
8888                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
8889                    tokio::task::yield_now().await;
8890                }
8891            })
8892            .await;
8893            let _ = module.stop().await;
8894            panic!(
8895                "assignment failure resumed a module; store touched={}",
8896                sentinel.exists()
8897            );
8898        }
8899        assert!(
8900            !sentinel.exists(),
8901            "refused module executed store-touch sentinel"
8902        );
8903        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
8904    }
8905
8906    #[cfg(windows)]
8907    #[tokio::test]
8908    async fn windows_nested_daemon_fixture() {
8909        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
8910            return;
8911        };
8912        let mut spec = spec();
8913        spec.env.push((
8914            "SUBC_TEST_STORE_TOUCH".into(),
8915            sentinel.to_str().unwrap().into(),
8916        ));
8917        let supervisor =
8918            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8919        let module = supervisor
8920            .spawn(spec)
8921            .expect("per-module job must nest below daemon and host jobs");
8922        tokio::time::timeout(Duration::from_secs(10), async {
8923            while !PathBuf::from(&sentinel).exists() {
8924                tokio::task::yield_now().await;
8925            }
8926        })
8927        .await
8928        .unwrap();
8929        module.stop().await.unwrap();
8930    }
8931
8932    #[cfg(windows)]
8933    #[test]
8934    fn windows_module_job_nests_under_host_job() {
8935        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
8936        let sentinel = home.join("nested-store-touched");
8937        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8938        command
8939            .args([
8940                "--exact",
8941                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
8942                "--nocapture",
8943            ])
8944            .env("SUBC_TEST_NESTED_STORE", &sentinel);
8945        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8946        assert!(daemon.child.wait().unwrap().success());
8947        assert!(sentinel.exists(), "nested protocol:none module never ran");
8948    }
8949}
8950
8951#[cfg(target_os = "linux")]
8952pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
8953    use subc_cgroup::KillOutcome;
8954    match subc_cgroup::kill_module(placement, module_id) {
8955        KillOutcome::Killed => {}
8956        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
8957            debug!(
8958                module_id,
8959                "cgroup tree kill unavailable; using direct-child kill"
8960            );
8961        }
8962        KillOutcome::IoError { path, error } => {
8963            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
8964        }
8965    }
8966}
8967
8968/// Assign a freshly spawned Windows child without ever resuming on failure.
8969///
8970/// Steps 2 and 3 of the suspended-create contract: the job is created and the
8971/// child assigned **while it is still suspended** (step 1 is
8972/// `suspend_on_create_async` at the spawn site). The caller resumes only after
8973/// the child's roster admission is durable. Failed containment kills and reaps
8974/// the suspended child instead of running an unprotected module.
8975#[cfg(windows)]
8976fn contain_spawned_child(
8977    child: &mut Child,
8978    spec: &ModuleSpec,
8979) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
8980    let assignment = || {
8981        let job = subc_jobobject::JobObject::new()?;
8982        #[cfg(test)]
8983        spawn_record_tests::assignment_failpoint(spec)?;
8984        job.assign(child)?;
8985        Ok::<_, io::Error>(job)
8986    };
8987    match assignment() {
8988        Ok(job) => Ok(Some(job)),
8989        Err(source) => {
8990            kill_and_reap_rejected_child(child);
8991            Err(SuperviseError::Spawn {
8992                program: spec.program.clone(),
8993                source: io::Error::other(format!(
8994                    "Windows module job containment failed: {source}"
8995                )),
8996                cgroup_path: None,
8997            })
8998        }
8999    }
9000}
9001
9002/// Rejection is synchronous: do not return a spawn error while the child can
9003/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
9004fn kill_and_reap_rejected_child(child: &mut Child) {
9005    #[cfg(unix)]
9006    if let Some(pid) = child
9007        .id()
9008        .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9009    {
9010        let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9011    }
9012    let _ = child.start_kill();
9013    while matches!(child.try_wait(), Ok(None)) {
9014        std::thread::yield_now();
9015    }
9016}
9017
9018/// Resume a suspended child, killing it if it cannot be started.
9019///
9020/// A suspended process holds a pid and does nothing, so there is no useful
9021/// state to return: the caller gets an error and the spawn fails.
9022#[cfg(windows)]
9023fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9024    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9025        // Kill it here rather than leaving a suspended process for the caller
9026        // to notice; `kill_on_drop` would eventually do this, but the module
9027        // would have been reported as running in between.
9028        let _ = std::process::Command::new("taskkill.exe")
9029            .args(["/PID", &pid.to_string(), "/T", "/F"])
9030            .stdin(Stdio::null())
9031            .stdout(Stdio::null())
9032            .stderr(Stdio::null())
9033            .status();
9034        return Err(SuperviseError::Spawn {
9035            program: spec.program.clone(),
9036            source,
9037            cgroup_path: None,
9038        });
9039    }
9040    Ok(())
9041}
9042
9043#[cfg(target_os = "linux")]
9044fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9045    match placement.remove_module(module_id) {
9046        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9047        Err(error) => warn!(
9048            module_id,
9049            error = %error,
9050            "could not remove module cgroup after process exit; continuing teardown"
9051        ),
9052    }
9053}
9054
9055#[cfg(target_os = "linux")]
9056async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9057    // Reaping the direct child is not proof its descendants exited. End the
9058    // residual tree and wait for the kernel's population fact before rmdir;
9059    // otherwise a successful parent wait leaks a directory on each restart.
9060    if matches!(
9061        subc_cgroup::kill_module(Some(placement), module_id),
9062        subc_cgroup::KillOutcome::Killed
9063    ) {
9064        if let Ok(path) = placement.module_path(module_id) {
9065            while std::fs::read_to_string(path.join("cgroup.events"))
9066                .ok()
9067                .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9068            {
9069                sleep(Duration::from_millis(1)).await;
9070            }
9071        }
9072    }
9073    remove_module_cgroup(placement, module_id);
9074}
9075
9076#[cfg(target_os = "linux")]
9077fn apply_cgroup_placement(
9078    command: &mut Command,
9079    spec: &ModuleSpec,
9080    path: &std::path::Path,
9081) -> Result<(), SuperviseError> {
9082    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9083        module_id: spec.module_id.clone(),
9084        source,
9085    })
9086}
9087
9088fn capture_retention(spec: &ModuleSpec) -> Retention {
9089    let defaults = Retention::default();
9090    let value = |name: &str| {
9091        spec.env
9092            .iter()
9093            .rev()
9094            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9095    };
9096    Retention {
9097        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9098            .and_then(|value| value.parse().ok())
9099            .unwrap_or(defaults.max_file_mb),
9100        keep: value(CAPTURE_KEEP_ENV)
9101            .and_then(|value| value.parse().ok())
9102            .unwrap_or(defaults.keep),
9103        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9104            .and_then(|value| value.parse().ok())
9105            .unwrap_or(defaults.max_age_days),
9106    }
9107}
9108
9109/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
9110/// module's registration to the exact process the supervisor spawned.
9111fn generate_launch_nonce() -> Result<String, SuperviseError> {
9112    let mut bytes = [0u8; 32];
9113    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9114        reason: source.to_string(),
9115    })?;
9116    let mut hex = String::with_capacity(64);
9117    for b in bytes {
9118        use std::fmt::Write;
9119        let _ = write!(hex, "{b:02x}");
9120    }
9121    Ok(hex)
9122}
9123
9124/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
9125/// signal about how many leading bytes matched.
9126fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9127    if a.len() != b.len() {
9128        return false;
9129    }
9130    let mut diff = 0u8;
9131    for (x, y) in a.iter().zip(b.iter()) {
9132        diff |= x ^ y;
9133    }
9134    diff == 0
9135}
9136
9137/// The kernel's image after an acknowledged exec, shared by ordinary launches
9138/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
9139/// not identities inferred from a configured pathname.
9140#[cfg(not(windows))]
9141fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9142    let process = subc_os::Process::open(pid).ok().flatten()?;
9143    #[cfg(target_os = "linux")]
9144    return sample_after_linux_exec(
9145        || {
9146            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9147            let committed = linux_exec_committed(&stat);
9148            #[cfg(test)]
9149            spawn_image_tests::report_exec_state(committed);
9150            committed
9151        },
9152        || process.observe(),
9153    );
9154    #[cfg(not(target_os = "linux"))]
9155    process.observe()
9156}
9157
9158/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
9159/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
9160/// installation. Until it clears, /proc/pid/exe can still name the daemon.
9161#[cfg(target_os = "linux")]
9162fn linux_exec_committed(stat: &str) -> Option<bool> {
9163    const PF_FORKNOEXEC: u64 = 0x40;
9164    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9165    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9166}
9167
9168#[cfg(target_os = "linux")]
9169fn sample_after_linux_exec(
9170    mut exec_committed: impl FnMut() -> Option<bool>,
9171    observe: impl FnOnce() -> Option<subc_os::Observation>,
9172) -> Option<subc_os::Observation> {
9173    let deadline = std::time::Instant::now() + Duration::from_secs(1);
9174    while !exec_committed()? {
9175        // A failed or indefinitely stalled exec must not publish the inherited
9176        // image as authority to signal a future orphan. Missing identity keeps
9177        // the existing conservative orphan-sweep behavior.
9178        if std::time::Instant::now() >= deadline {
9179            return None;
9180        }
9181        std::thread::sleep(Duration::from_micros(50));
9182    }
9183    observe()
9184}
9185
9186#[cfg(all(test, target_os = "linux"))]
9187mod spawn_image_tests {
9188    use super::*;
9189    use std::cell::{Cell, RefCell};
9190
9191    enum ExecEvent {
9192        Uncommitted,
9193        Image(Option<subc_os::Observation>),
9194    }
9195
9196    thread_local! {
9197        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9198    }
9199
9200    pub(super) fn report_exec_state(committed: Option<bool>) {
9201        if committed == Some(false) {
9202            EXEC_EVENTS.with(|events| {
9203                if let Some(events) = events.borrow_mut().take() {
9204                    let _ = events.send(ExecEvent::Uncommitted);
9205                }
9206            });
9207        }
9208    }
9209
9210    #[test]
9211    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9212        use std::io::{Read, Write};
9213        use std::os::fd::AsRawFd;
9214
9215        let descriptor = std::fs::File::open("/dev/null").unwrap();
9216        let mut command = std::process::Command::new("/bin/sleep");
9217        command.arg("30");
9218        let (mut ready, mut release) =
9219            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9220                .unwrap();
9221        let spawn = std::thread::spawn(move || command.spawn());
9222        let mut message = [0u8; 8];
9223        ready.read_exact(&mut message).unwrap();
9224        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9225        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9226        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9227        let (events, received) = std::sync::mpsc::channel();
9228        let observer = std::thread::spawn(move || {
9229            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9230            let image = observe_spawned_image(pid);
9231            let _ = events.send(ExecEvent::Image(image));
9232        });
9233        // Release only once the production observer has seen the uncommitted
9234        // exec, or has incorrectly returned the inherited image. Either way,
9235        // release and reap before asserting so a regression cannot leak a child.
9236        let first = received.recv_timeout(Duration::from_secs(5));
9237        release.write_all(&[1]).unwrap();
9238        let mut child = spawn.join().unwrap().unwrap();
9239        let after = match first {
9240            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9241                Ok(ExecEvent::Image(image)) => image,
9242                _ => None,
9243            },
9244            Ok(ExecEvent::Image(image)) => image,
9245            Err(_) => None,
9246        };
9247        observer.join().unwrap();
9248        child.kill().unwrap();
9249        child.wait().unwrap();
9250
9251        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9252        assert_eq!(
9253            inherited.unwrap().executable,
9254            subc_os::file_identity(&std::env::current_exe().unwrap())
9255        );
9256        assert_eq!(
9257            after.unwrap().executable,
9258            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9259        );
9260    }
9261
9262    #[test]
9263    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9264        let committed = Cell::new(false);
9265        let inherited = subc_os::Observation {
9266            start_time: 23,
9267            executable: Some(subc_os::FileIdentity {
9268                device: 29,
9269                inode: 31,
9270            }),
9271        };
9272        let module = subc_os::Observation {
9273            start_time: 23,
9274            executable: Some(subc_os::FileIdentity {
9275                device: 29,
9276                inode: 37,
9277            }),
9278        };
9279        let polls = Cell::new(0);
9280        let image = sample_after_linux_exec(
9281            || {
9282                polls.set(polls.get() + 1);
9283                // Hold the inherited image through the first poll, as in the
9284                // posix_spawn parent-wake window, then commit the module image.
9285                committed.set(polls.get() == 2);
9286                Some(committed.get())
9287            },
9288            || Some(if committed.get() { module } else { inherited }),
9289        );
9290        assert_eq!(image, Some(module));
9291        assert_eq!(polls.get(), 2);
9292    }
9293
9294    #[test]
9295    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9296        assert_eq!(
9297            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9298            Some(false)
9299        );
9300        assert_eq!(
9301            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9302            Some(true)
9303        );
9304        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9305        assert_eq!(linux_exec_committed("17 (a) R"), None);
9306    }
9307
9308    #[test]
9309    fn unreadable_exec_state_never_samples_an_inherited_image() {
9310        assert_eq!(
9311            sample_after_linux_exec(
9312                || None,
9313                || panic!("must not sample before exec confirmation")
9314            ),
9315            None
9316        );
9317    }
9318
9319    #[test]
9320    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9321        assert_eq!(
9322            sample_after_linux_exec(
9323                || Some(false),
9324                || panic!("must not sample after an exec timeout")
9325            ),
9326            None
9327        );
9328    }
9329}
9330
9331/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
9332/// different executable after the no-delete capture was taken.
9333#[cfg(windows)]
9334fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9335    let mut program = spec.program.clone();
9336    if program.is_absolute() || program.components().count() > 1 {
9337        // An explicit path can name a PE image with no extension. Try that
9338        // exact file first, as Command does, before falling back to .exe.
9339        match subc_os::ExecutableCapture::open(&program) {
9340            Ok(capture) => return Ok(capture),
9341            Err(error)
9342                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9343            Err(error) => return Err(error),
9344        }
9345        program.set_extension("exe");
9346        return subc_os::ExecutableCapture::open(&program);
9347    }
9348    if program.extension().is_none() {
9349        program.set_extension("exe");
9350    }
9351    let custom_path = spec
9352        .env
9353        .iter()
9354        .rev()
9355        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9356    let mut directories = Vec::new();
9357    // These are Windows' default executable search locations. A configured
9358    // PATH still overrides the inherited PATH, not the system directories.
9359    if let Some(parent) = std::env::current_exe()?.parent() {
9360        directories.push(parent.to_path_buf());
9361    }
9362    directories.push(std::env::current_dir()?);
9363    if let Some(windows) = std::env::var_os("SystemRoot") {
9364        let windows = PathBuf::from(windows);
9365        directories.push(windows.join("System32"));
9366        directories.push(windows.clone());
9367    }
9368    let path = custom_path
9369        .map(|(_, value)| std::ffi::OsString::from(value))
9370        .or_else(|| std::env::var_os("PATH"));
9371    if let Some(path) = path {
9372        directories.extend(std::env::split_paths(&path));
9373    }
9374    for directory in directories {
9375        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9376            Ok(capture) => return Ok(capture),
9377            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9378            Err(error) => return Err(error),
9379        }
9380    }
9381    Err(io::Error::new(
9382        io::ErrorKind::NotFound,
9383        "resolved executable not found",
9384    ))
9385}
9386
9387fn spawn_and_mark_running(
9388    spec: &ModuleSpec,
9389    runtime: &SupervisorRuntimeConfig,
9390    snapshot: &SharedSnapshot,
9391) -> Result<SupervisedChild, SuperviseError> {
9392    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9393    let child = spawn_child(
9394        spec,
9395        runtime.connection_file_path.as_deref(),
9396        runtime.supervisor_handle.as_ref(),
9397        &runtime.stderr_ring,
9398        runtime.capture_logs_dir.as_deref(),
9399        &runtime.child_roster,
9400        #[cfg(target_os = "linux")]
9401        runtime.cgroup_placement.as_ref(),
9402    )?;
9403    set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9404    if let Some(spawn) = activity_spawn {
9405        spawn.complete(snapshot);
9406    }
9407    Ok(child)
9408}
9409
9410enum RegistrationWaitOutcome {
9411    Registered,
9412    Exited(ExitReport),
9413    TimedOut,
9414}
9415
9416struct ReloadRegistrationFailure {
9417    exit_report: ExitReport,
9418    reason: String,
9419}
9420
9421#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9422enum BusyGaugeObservation {
9423    Quiescent,
9424    Busy,
9425    Omitted,
9426}
9427
9428fn busy_gauge_observation(metrics: Option<&Value>, gauges: &[String]) -> BusyGaugeObservation {
9429    let Some(metrics) = metrics.and_then(Value::as_object) else {
9430        return BusyGaugeObservation::Omitted;
9431    };
9432    let mut sum = 0u128;
9433    for gauge in gauges {
9434        let Some(value) = metrics.get(gauge) else {
9435            return BusyGaugeObservation::Omitted;
9436        };
9437        let Some(value) = value.as_u64() else {
9438            return BusyGaugeObservation::Busy;
9439        };
9440        sum = sum.saturating_add(u128::from(value));
9441    }
9442    if sum == 0 {
9443        BusyGaugeObservation::Quiescent
9444    } else {
9445        BusyGaugeObservation::Busy
9446    }
9447}
9448
9449fn declared_busy_gauges(
9450    registry: &Registry,
9451    module_id: &str,
9452) -> Result<Vec<String>, SuperviseError> {
9453    busy_gauges_of(
9454        registry
9455            .get_module(module_id)
9456            .map_err(SuperviseError::Registry)?,
9457    )
9458}
9459
9460/// [`declared_busy_gauges`] for the registration a connection holds, in any
9461/// slot: after cutover the incumbent is no longer the id's active
9462/// registration, and its own manifest is the one that names its gauges.
9463fn declared_busy_gauges_for_connection(
9464    registry: &Registry,
9465    connection_id: ConnectionId,
9466) -> Result<Vec<String>, SuperviseError> {
9467    busy_gauges_of(
9468        registry
9469            .get_module_by_connection(connection_id)
9470            .map_err(SuperviseError::Registry)?,
9471    )
9472}
9473
9474fn busy_gauges_of(
9475    registration: Option<crate::registry::ModuleRegistration>,
9476) -> Result<Vec<String>, SuperviseError> {
9477    let Some(registration) = registration else {
9478        return Ok(Vec::new());
9479    };
9480    let Some(self_signals) = registration.manifest.self_signals else {
9481        return Ok(Vec::new());
9482    };
9483
9484    let mut gauges = Vec::new();
9485    for declaration in self_signals {
9486        if declaration.kind != SelfSignalKind::Busy {
9487            continue;
9488        }
9489        match declaration.anchored_to {
9490            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9491                gauges.extend(declared)
9492            }
9493            _ => {
9494                // An invalid Busy anchor is fail-safe: the empty name cannot be
9495                // present in a conforming health report, so this drain stays busy.
9496                gauges.push(String::new());
9497            }
9498        }
9499    }
9500    Ok(gauges)
9501}
9502
9503/// Wait for `endpoint` to have nothing in flight and, when the module declares
9504/// busy gauges, for a health probe to report them quiet. The probe is addressed
9505/// by `scope`: a swap's superseded incumbent must be asked about its own
9506/// gauges, and by module id the probe would reach the promoted candidate.
9507/// While the gauges read busy, the probe repeats every
9508/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
9509async fn wait_for_forwarding_quiescence(
9510    forwarding: &ForwardingTable,
9511    module_id: &str,
9512    runtime: &SupervisorRuntimeConfig,
9513    endpoint: crate::ModuleEndpointId,
9514    deadline: Instant,
9515    busy_gauges: &[String],
9516    scope: DrainScope,
9517) -> Result<bool, SuperviseError> {
9518    let mut gauges_quiescent = busy_gauges.is_empty();
9519    let mut next_probe_at = Instant::now();
9520    let mut omission_counted = false;
9521
9522    loop {
9523        let now = Instant::now();
9524        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9525            let report = match scope {
9526                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9527                DrainScope::Endpoint(endpoint) => {
9528                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9529                }
9530            };
9531            gauges_quiescent = match report {
9532                Ok(report) => match busy_gauge_observation(report.metrics.as_ref(), busy_gauges) {
9533                    BusyGaugeObservation::Quiescent => true,
9534                    BusyGaugeObservation::Busy => false,
9535                    BusyGaugeObservation::Omitted => {
9536                        if !omission_counted {
9537                            forwarding
9538                                .counters()
9539                                .increment_drains_with_undeclared_gauge();
9540                            omission_counted = true;
9541                        }
9542                        false
9543                    }
9544                },
9545                Err(err) => {
9546                    warn!(
9547                        module_id,
9548                        error = %err,
9549                        "drain health.check did not produce declared busy gauges; treating module as busy"
9550                    );
9551                    false
9552                }
9553            };
9554            // Counted from the probe's completion: the probe above was awaited,
9555            // so no second probe can start while one is still outstanding.
9556            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
9557        }
9558
9559        let in_flight = forwarding
9560            .endpoint_in_flight_count(endpoint)
9561            .map_err(SuperviseError::Forwarding)?;
9562        if in_flight == 0 && gauges_quiescent {
9563            return Ok(true);
9564        }
9565
9566        let now = Instant::now();
9567        if now >= deadline {
9568            return Ok(false);
9569        }
9570        let mut wait = deadline
9571            .saturating_duration_since(now)
9572            .min(REGISTRY_RELEASE_POLL);
9573        if !busy_gauges.is_empty() {
9574            wait = wait.min(next_probe_at.saturating_duration_since(now));
9575        }
9576        sleep(wait).await;
9577    }
9578}
9579
9580/// The wait between drain busy-gauge probes: the larger of
9581/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
9582/// drain's own in-flight poll interval.
9583fn drain_busy_reprobe_interval() -> Duration {
9584    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
9585}
9586
9587/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
9588///
9589/// `Ok` is always honest and passed straight through -- the wait actually measured
9590/// in-flight state. `Err` means the wait produced no measurement at all (the
9591/// forwarding table's lock was poisoned), so `false` is reported as the one honest
9592/// constant: the drain did not complete. Never recomputed from route state, never a
9593/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
9594fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
9595    match wait_result {
9596        Ok(drained) => *drained,
9597        Err(_) => false,
9598    }
9599}
9600
9601fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
9602    for released in released_routes {
9603        let frame = match Frame::build_with_version(
9604            released.negotiated_ver,
9605            FrameType::Goodbye,
9606            control_flags(),
9607            released.channel,
9608            released.epoch,
9609            0,
9610            Vec::new(),
9611        ) {
9612            Ok(frame) => frame,
9613            Err(err) => {
9614                warn!(
9615                    route_channel = released.channel,
9616                    error = %err,
9617                    "failed to build supervisor drain route GOODBYE frame"
9618                );
9619                continue;
9620            }
9621        };
9622        if !released.close_on_delivery_failure() {
9623            crate::forwarding::send_module_route_goodbye(
9624                &forwarding.counters(),
9625                &released.sink,
9626                frame,
9627                released.module_id.as_deref(),
9628                "supervisor drain",
9629            );
9630            continue;
9631        }
9632        if let Err(err) = released.sink.try_send(frame) {
9633            warn!(
9634                target_connection_id = released.connection_id.get(),
9635                route_channel = released.channel,
9636                error = %err,
9637                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
9638            );
9639            let _ = forwarding.escalate_client_delivery_failure(
9640                released.connection_id,
9641                released.channel,
9642                released.epoch,
9643                CloseReason::new(
9644                    "route_goodbye_delivery_failed",
9645                    format!(
9646                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
9647                        released.channel
9648                    ),
9649                ),
9650                crate::forwarding::UndeliveredFrame {
9651                    module_id: released.module_id.as_deref(),
9652                    sink: &released.sink,
9653                },
9654            );
9655        }
9656    }
9657}
9658
9659fn send_module_draining(
9660    module_id: &str,
9661    reason: RouteCloseReason,
9662    deadline_ms: u64,
9663    target: &ModuleDrainTarget,
9664) {
9665    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
9666        reason,
9667        deadline_ms,
9668    }) {
9669        Ok(body) => body,
9670        Err(err) => {
9671            warn!(
9672                module_id,
9673                error = %err,
9674                "failed to encode module draining command"
9675            );
9676            return;
9677        }
9678    };
9679    let frame = match Frame::build_with_version(
9680        target.negotiated_ver,
9681        FrameType::Push,
9682        control_flags(),
9683        0,
9684        0,
9685        0,
9686        body,
9687    ) {
9688        Ok(frame) => frame,
9689        Err(err) => {
9690            warn!(
9691                module_id,
9692                error = %err,
9693                "failed to build module draining command frame"
9694            );
9695            return;
9696        }
9697    };
9698    if let Err(err) = target.sink.try_send(frame) {
9699        warn!(
9700            module_id,
9701            target_connection_id = target.endpoint.connection_id.get(),
9702            error = %err,
9703            "module draining command was not delivered to peer"
9704        );
9705    }
9706}
9707
9708/// The channel-0 GOODBYE that tells a module its stop is planned.
9709fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
9710    match Frame::build_with_version(
9711        negotiated_ver,
9712        FrameType::Goodbye,
9713        control_flags(),
9714        0,
9715        0,
9716        0,
9717        Vec::new(),
9718    ) {
9719        Ok(frame) => Some(frame),
9720        Err(err) => {
9721            warn!(
9722                module_id,
9723                error = %err,
9724                "failed to build module GOODBYE frame"
9725            );
9726            None
9727        }
9728    }
9729}
9730
9731/// Send every registered module connection its module GOODBYE at daemon
9732/// shutdown, then request that connection's close.
9733///
9734/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
9735/// before EOF, so the GOODBYE must reach the socket before the close. A close
9736/// request does not wait for the connection's queued frames: its writer gets a
9737/// bounded grace after the close, is aborted if it overruns it, and the daemon
9738/// process may exit before that grace ends. So with `wait_for_flush`, each
9739/// connection is closed only after its writer has acknowledged writing the
9740/// GOODBYE, or once a short shared budget runs out, so one module that is not
9741/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
9742/// are only queued, for a shutdown the operator has told to stop waiting.
9743/// A connection that is already gone is skipped.
9744#[cfg(unix)]
9745async fn send_module_goodbyes_for_daemon_shutdown(
9746    forwarding: &Arc<ForwardingTable>,
9747    reason: &CloseReason,
9748    wait_for_flush: bool,
9749) {
9750    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
9751    let targets = match forwarding.module_connections() {
9752        Ok(targets) => targets,
9753        Err(err) => {
9754            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
9755            return;
9756        }
9757    };
9758    let deadline = Instant::now() + GOODBYE_BUDGET;
9759    let mut sends = tokio::task::JoinSet::new();
9760    for target in targets {
9761        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
9762            continue;
9763        };
9764        if !wait_for_flush {
9765            if let Err(err) = target.sink.try_send(frame) {
9766                debug!(
9767                    module_id = %target.module_id,
9768                    error = %err,
9769                    "shutdown module GOODBYE was not queued"
9770                );
9771            }
9772            continue;
9773        }
9774        let forwarding = Arc::clone(forwarding);
9775        let reason = reason.clone();
9776        sends.spawn(async move {
9777            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
9778                Ok(Ok(())) => {}
9779                Ok(Err(err)) => debug!(
9780                    module_id = %target.module_id,
9781                    error = %err,
9782                    "module connection closed before its shutdown GOODBYE was written"
9783                ),
9784                Err(_) => warn!(
9785                    module_id = %target.module_id,
9786                    budget = ?GOODBYE_BUDGET,
9787                    "shutdown module GOODBYE was not written within its budget; closing anyway"
9788                ),
9789            }
9790            forwarding.request_connection_close(target.endpoint.connection_id, reason);
9791        });
9792    }
9793    // Every task ends by the shared deadline, so this wait is bounded too.
9794    while sends.join_next().await.is_some() {}
9795}
9796
9797fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
9798    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
9799        return;
9800    };
9801    if let Err(err) = target.sink.try_send(frame) {
9802        warn!(
9803            module_id,
9804            target_connection_id = target.endpoint.connection_id.get(),
9805            error = %err,
9806            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
9807        );
9808        forwarding.request_connection_close(
9809            target.endpoint.connection_id,
9810            CloseReason::new(
9811                "module_goodbye_delivery_failed",
9812                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
9813            ),
9814        );
9815    }
9816}
9817
9818#[derive(Clone, Copy)]
9819struct ForwardingDrainContext<'a> {
9820    spec: &'a ModuleSpec,
9821    runtime: &'a SupervisorRuntimeConfig,
9822    registry: &'a Registry,
9823    scope: DrainScope,
9824}
9825
9826/// Which process a forwarding drain addresses.
9827#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9828enum DrainScope {
9829    /// Whatever endpoint is active for the module id: every plain stop,
9830    /// restart and reload. Also moves the module's state to `Draining`.
9831    Active,
9832    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
9833    /// module id would resolve to the promoted candidate and leave neither
9834    /// process routable. The module's state is left alone, since the promoted
9835    /// candidate is what it describes and that process is running.
9836    Endpoint(crate::ModuleEndpointId),
9837}
9838
9839/// Whether a child being drained has already been asked to stop by the time
9840/// its drain wait starts.
9841///
9842/// The drain wait is the same budget whatever this says. What it decides is
9843/// whether the supervisor must ask by signal before that wait begins: a child
9844/// that nobody asked will sit out the whole budget and then be SIGKILLed,
9845/// healthy or not.
9846#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9847enum StopNotice {
9848    /// The module was sent `module.draining` and a module GOODBYE over its own
9849    /// registered connection, and stops itself.
9850    SentOverConnection,
9851    /// The forwarding drain found no registered connection for the module: a
9852    /// subc child spawned moments ago that has not sent HELLO yet, or a
9853    /// `protocol: "none"` child, which never registers.
9854    NoConnection,
9855    /// This path sends nothing over the module's connection: the supervisor has
9856    /// no forwarding table, or the caller stops the child without a forwarding
9857    /// drain.
9858    NotSent,
9859}
9860
9861async fn begin_forwarding_drain(
9862    spec: &ModuleSpec,
9863    runtime: &SupervisorRuntimeConfig,
9864    registry: &Registry,
9865    snapshot: &SharedSnapshot,
9866    enabled: Option<bool>,
9867    reason: RouteCloseReason,
9868) -> Result<StopNotice, SuperviseError> {
9869    let Some(forwarding) = runtime.forwarding.as_ref() else {
9870        return Err(SuperviseError::ReloadUnavailable {
9871            module_id: spec.module_id.clone(),
9872            reason: "supervisor was not configured with a forwarding table".to_string(),
9873        });
9874    };
9875
9876    begin_forwarding_drain_with(
9877        forwarding,
9878        ForwardingDrainContext {
9879            spec,
9880            runtime,
9881            registry,
9882            scope: DrainScope::Active,
9883        },
9884        snapshot,
9885        enabled,
9886        reason,
9887        runtime.drain_timeout,
9888    )
9889    .await
9890}
9891
9892async fn begin_forwarding_drain_if_configured(
9893    spec: &ModuleSpec,
9894    runtime: &SupervisorRuntimeConfig,
9895    registry: &Registry,
9896    snapshot: &SharedSnapshot,
9897    enabled: Option<bool>,
9898    reason: RouteCloseReason,
9899) -> Result<StopNotice, SuperviseError> {
9900    begin_forwarding_drain_with_timeout(
9901        spec,
9902        runtime,
9903        registry,
9904        snapshot,
9905        enabled,
9906        reason,
9907        runtime.drain_timeout,
9908    )
9909    .await
9910}
9911
9912/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
9913/// budget, for paths where the operator overrides the module's configured one
9914/// (`supervisor.restart{drain_timeout_ms}`).
9915async fn begin_forwarding_drain_with_timeout(
9916    spec: &ModuleSpec,
9917    runtime: &SupervisorRuntimeConfig,
9918    registry: &Registry,
9919    snapshot: &SharedSnapshot,
9920    enabled: Option<bool>,
9921    reason: RouteCloseReason,
9922    drain_timeout: Duration,
9923) -> Result<StopNotice, SuperviseError> {
9924    let Some(forwarding) = runtime.forwarding.as_ref() else {
9925        return Ok(StopNotice::NotSent);
9926    };
9927
9928    begin_forwarding_drain_with(
9929        forwarding,
9930        ForwardingDrainContext {
9931            spec,
9932            runtime,
9933            registry,
9934            scope: DrainScope::Active,
9935        },
9936        snapshot,
9937        enabled,
9938        reason,
9939        drain_timeout,
9940    )
9941    .await
9942}
9943
9944async fn begin_forwarding_drain_with(
9945    forwarding: &ForwardingTable,
9946    context: ForwardingDrainContext<'_>,
9947    snapshot: &SharedSnapshot,
9948    enabled: Option<bool>,
9949    reason: RouteCloseReason,
9950    drain_timeout: Duration,
9951) -> Result<StopNotice, SuperviseError> {
9952    let ForwardingDrainContext {
9953        spec,
9954        runtime,
9955        registry,
9956        scope,
9957    } = context;
9958    debug_assert_ne!(reason, RouteCloseReason::Crash);
9959    let terminal = matches!(reason, RouteCloseReason::Disable);
9960    let drain_started_at = Instant::now();
9961    let drain_deadline = drain_started_at + drain_timeout;
9962    let deadline_ms =
9963        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
9964    let busy_gauges = match scope {
9965        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
9966        DrainScope::Endpoint(endpoint) => {
9967            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
9968        }
9969    };
9970
9971    // Admission gate first: route.open/commit and route REQUEST admission are closed
9972    // before the first quiescence check, so the outstanding count can only fall.
9973    let gate_started = Instant::now();
9974    let drain_target = match scope {
9975        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
9976        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
9977    }
9978    .map_err(SuperviseError::Forwarding)?;
9979    // The instant admission closed, and how long taking the forwarding write
9980    // lock to close it took. The timeout line reports only the quiescence
9981    // wait, so without this a drain that started late looked like one that
9982    // started on time.
9983    info!(
9984        module_id = %spec.module_id,
9985        ?reason,
9986        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
9987        connected = drain_target.is_some(),
9988        "module drain began; route admission closed"
9989    );
9990    if scope == DrainScope::Active {
9991        update_snapshot(snapshot, Some(&spec.module_id), |state| {
9992            state.state = ModuleState::Draining;
9993            state.draining_to_replace =
9994                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
9995            if let Some(enabled) = enabled {
9996                state.enabled = enabled;
9997            }
9998        })?;
9999    }
10000
10001    let Some(target) = drain_target.as_ref() else {
10002        // Nothing was sent: the module has no registered connection to carry
10003        // `module.draining` or a GOODBYE. The caller must not assume the child
10004        // was asked to stop.
10005        return Ok(StopNotice::NoConnection);
10006    };
10007    {
10008        send_module_draining(&spec.module_id, reason, deadline_ms, target);
10009        #[cfg(feature = "test-support")]
10010        forwarding.wait_before_route_closing_for_test().await;
10011        let routes = forwarding
10012            .endpoint_routes(target.endpoint)
10013            .map_err(SuperviseError::Forwarding)?;
10014        let routes_notified = routes.len();
10015        crate::control::send_route_control_pushes(
10016            forwarding,
10017            routes.clone(),
10018            ClientControlPush::RouteClosing {
10019                module_id: spec.module_id.clone(),
10020                channels: Vec::new(),
10021                reason,
10022            },
10023        );
10024        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10025
10026        // `route.closing` was just sent above: from here on every return path,
10027        // including an early one, MUST send `route.closed` before propagating
10028        // anything else. A client holds `closing` as a promise that a verdict is
10029        // coming; leaving early without `closed` strands it waiting forever, since
10030        // `closing` carries no timeout of its own.
10031        let wait_result = wait_for_forwarding_quiescence(
10032            forwarding,
10033            &spec.module_id,
10034            runtime,
10035            target.endpoint,
10036            drain_deadline,
10037            &busy_gauges,
10038            scope,
10039        )
10040        .await;
10041        let drained = drained_after_quiescence_wait(&wait_result);
10042        if let Err(err) = &wait_result {
10043            error!(
10044                module_id = %spec.module_id,
10045                ?reason,
10046                error = %err,
10047                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10048            );
10049        } else if !drained {
10050            // Name what the drain waited on. Without it the line says only that
10051            // something did not settle, and "one wedged call" and "every
10052            // session's held stream" read the same; the first is a module bug,
10053            // the second is a module that should end its streams on
10054            // module.draining. Read before teardown releases the routes.
10055            let holdouts = forwarding
10056                .endpoint_drain_holdouts(target.endpoint)
10057                .unwrap_or_default();
10058            warn!(
10059                module_id = %spec.module_id,
10060                waited = ?drain_timeout,
10061                ?reason,
10062                held_requests = holdouts.requests,
10063                held_routes = holdouts.routes,
10064                total_routes = holdouts.total_routes,
10065                top_connections = ?holdouts.top_connections,
10066                // `module_channel:corr`, so the module can find each held request
10067                // in its own log; capped, so `held_requests` is the full count.
10068                held = %holdouts
10069                    .held
10070                    .iter()
10071                    .map(|(channel, corr)| format!("{channel}:{corr}"))
10072                    .collect::<Vec<_>>()
10073                    .join(","),
10074                "route drain timed out before request quiescence; forcing teardown"
10075            );
10076        }
10077        crate::control::send_route_control_pushes(
10078            forwarding,
10079            routes,
10080            ClientControlPush::RouteClosed {
10081                module_id: spec.module_id.clone(),
10082                channels: Vec::new(),
10083                reason,
10084                drained,
10085                abandoned: target.abandoned_bindings.len() as u32,
10086                excluded_subscriptions: target.excluded_subscriptions,
10087                terminal: Some(terminal),
10088            },
10089        );
10090        wait_result?;
10091
10092        // `route.closed` has now been sent unconditionally above. From here the
10093        // remaining steps are cleanup (route + module GOODBYE) rather than a
10094        // promise the client is waiting on, but a lock-poisoned
10095        // `release_module_endpoint_routes` would otherwise skip the module
10096        // GOODBYE silently too -- send it before propagating the error.
10097        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10098            Ok(routes) => routes,
10099            Err(err) => {
10100                warn!(
10101                    module_id = %spec.module_id,
10102                    ?reason,
10103                    error = %err,
10104                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10105                );
10106                send_module_goodbye(&spec.module_id, forwarding, target);
10107                return Err(SuperviseError::Forwarding(err));
10108            }
10109        };
10110        let route_goodbye_count = released_routes.len();
10111        send_route_goodbyes(forwarding, released_routes);
10112        send_module_goodbye(&spec.module_id, forwarding, target);
10113
10114        // The drain's happy path was previously silent: every emission above is
10115        // best-effort with only its failure arm logged, so "were consumers told"
10116        // was unprovable from the daemon log (surfaced by a 30-minute consumer
10117        // hang where the open question was exactly whether teardown notice went
10118        // out). One summary line makes that class decidable in one grep.
10119        info!(
10120            module_id = %spec.module_id,
10121            ?reason,
10122            routes_notified,
10123            route_goodbyes = route_goodbye_count,
10124            abandoned_reservations = target.abandoned_bindings.len(),
10125            excluded_subscriptions = target.excluded_subscriptions,
10126            drained,
10127            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10128        );
10129    }
10130
10131    Ok(StopNotice::SentOverConnection)
10132}
10133
10134/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
10135/// the only slot a plain (non-swap) spawn can register into.
10136async fn wait_for_registration_after_reload(
10137    registry: &Registry,
10138    module_id: &str,
10139    snapshot: &SharedSnapshot,
10140    child: &mut SupervisedChild,
10141    wait: Duration,
10142) -> Result<RegistrationWaitOutcome, SuperviseError> {
10143    wait_for_slot_registration(
10144        registry,
10145        crate::registry::RegistrationSlot::Active(module_id),
10146        module_id,
10147        snapshot,
10148        child,
10149        wait,
10150    )
10151    .await
10152}
10153
10154/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
10155///
10156/// Keyed on the slot rather than the bare module id because during a swap the
10157/// id's active slot is already held by the incumbent: an id-keyed wait would
10158/// report the incumbent's registration as the candidate's and a candidate that
10159/// never registers would look registered. A swap candidate waits on
10160/// `crate::registry::RegistrationSlot::Candidate`.
10161async fn wait_for_slot_registration(
10162    registry: &Registry,
10163    slot: crate::registry::RegistrationSlot<'_>,
10164    module_id: &str,
10165    snapshot: &SharedSnapshot,
10166    child: &mut SupervisedChild,
10167    wait: Duration,
10168) -> Result<RegistrationWaitOutcome, SuperviseError> {
10169    let deadline = Instant::now() + wait;
10170    loop {
10171        if registry
10172            .registration(slot)
10173            .map_err(SuperviseError::Registry)?
10174            .is_some()
10175        {
10176            return Ok(RegistrationWaitOutcome::Registered);
10177        }
10178
10179        let now = Instant::now();
10180        if now >= deadline {
10181            return Ok(RegistrationWaitOutcome::TimedOut);
10182        }
10183        let remaining = deadline.saturating_duration_since(now);
10184        let poll = remaining.min(REGISTRY_RELEASE_POLL);
10185
10186        tokio::select! {
10187            wait_result = child.wait() => {
10188                let status = wait_result.map_err(|source| SuperviseError::Wait {
10189                    module_id: module_id.to_string(),
10190                    source,
10191                })?;
10192                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10193                    snapshot,
10194                    child,
10195                    &status,
10196                )));
10197            }
10198            _ = sleep(poll) => {}
10199        }
10200    }
10201}
10202
10203fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10204    // A replacement process that exits before HELLO did not provide service, even
10205    // if it used status 0. Count it against the restart cap as a new-binary failure.
10206    if exit_report.kind != ExitKind::DeliberateSeverance {
10207        exit_report.kind = ExitKind::Crash;
10208    }
10209    exit_report
10210}
10211
10212async fn handle_reload_child_registration_failure(
10213    spec: &ModuleSpec,
10214    runtime: &SupervisorRuntimeConfig,
10215    registry: &Registry,
10216    process_liveness: &SupervisorProcessLiveness,
10217    snapshot: &SharedSnapshot,
10218    _child: &mut Option<SupervisedChild>,
10219    failure: ReloadRegistrationFailure,
10220) -> Result<(), SuperviseError> {
10221    let ReloadRegistrationFailure {
10222        exit_report,
10223        reason,
10224    } = failure;
10225    match on_child_exit(
10226        spec,
10227        runtime.restart_policy,
10228        registry,
10229        snapshot,
10230        &runtime.terminal_ring,
10231        &runtime.spawn_events,
10232        &runtime.child_roster,
10233        exit_report,
10234    )
10235    .await
10236    {
10237        NextAction::Stop {
10238            registration_released,
10239        } => {
10240            if registration_released {
10241                process_liveness.untrack_if_current(&spec.module_id, snapshot);
10242            }
10243        }
10244        NextAction::Restart { schedule } => {
10245            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10246                schedule.delay
10247            });
10248            if let Some(schedule) = schedule {
10249                log_crash_respawn(&spec.module_id, schedule);
10250            }
10251            schedule_respawn(
10252                runtime,
10253                snapshot,
10254                &spec.module_id,
10255                delay,
10256                RespawnKind::Spawn,
10257            )?;
10258        }
10259    }
10260    Err(SuperviseError::ReloadFailed {
10261        module_id: spec.module_id.clone(),
10262        reason,
10263    })
10264}
10265
10266async fn handle_reload_spawn_failure(
10267    spec: &ModuleSpec,
10268    runtime: &SupervisorRuntimeConfig,
10269    process_liveness: &SupervisorProcessLiveness,
10270    snapshot: &SharedSnapshot,
10271    _child: &mut Option<SupervisedChild>,
10272    reason: String,
10273) -> Result<(), SuperviseError> {
10274    let now = Instant::now();
10275    let mut schedule = None;
10276    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10277        clear_current_process_facts(state);
10278        if state.enabled {
10279            schedule = state.next_crash_restart(&runtime.restart_policy, now);
10280            state.state = if schedule.is_some() {
10281                ModuleState::Restarting
10282            } else {
10283                ModuleState::Failed
10284            };
10285        } else {
10286            state.state = ModuleState::Disabled;
10287        }
10288    })?;
10289    if let Some(schedule) = schedule {
10290        schedule_respawn(
10291            runtime,
10292            snapshot,
10293            &spec.module_id,
10294            schedule.delay,
10295            RespawnKind::Spawn,
10296        )?;
10297    } else {
10298        process_liveness.untrack_if_current(&spec.module_id, snapshot);
10299    }
10300    Err(SuperviseError::ReloadFailed {
10301        module_id: spec.module_id.clone(),
10302        reason,
10303    })
10304}
10305
10306fn control_flags() -> Flags {
10307    Flags::new(false, Priority::Passive, false)
10308}
10309
10310#[allow(clippy::too_many_arguments)]
10311async fn drain_optional_child(
10312    module_id: &str,
10313    protocol: ModuleProtocol,
10314    stop_notice: StopNotice,
10315    registry: &Registry,
10316    forwarding: Option<&ForwardingTable>,
10317    snapshot: &SharedSnapshot,
10318    terminal_ring: &Arc<Mutex<TerminalRing>>,
10319    spawn_events: &SpawnEventFeed,
10320    child: &mut Option<SupervisedChild>,
10321    drain_timeout: Duration,
10322    final_state: ModuleState,
10323    enabled: Option<bool>,
10324) -> Result<(), SuperviseError> {
10325    if let Some(child) = child.take() {
10326        drain_child_to_state(
10327            module_id,
10328            protocol,
10329            stop_notice,
10330            registry,
10331            forwarding,
10332            snapshot,
10333            terminal_ring,
10334            spawn_events,
10335            child,
10336            drain_timeout,
10337            final_state,
10338            enabled,
10339        )
10340        .await
10341    } else {
10342        update_snapshot(snapshot, Some(module_id), |state| {
10343            state.state = final_state;
10344            if let Some(enabled) = enabled {
10345                state.enabled = enabled;
10346            }
10347            clear_current_process_facts(state);
10348        })?;
10349        release_dead_registration(registry, forwarding, snapshot, module_id).await
10350    }
10351}
10352
10353#[allow(clippy::too_many_arguments)]
10354async fn drain_child_to_state(
10355    module_id: &str,
10356    _protocol: ModuleProtocol,
10357    stop_notice: StopNotice,
10358    registry: &Registry,
10359    forwarding: Option<&ForwardingTable>,
10360    snapshot: &SharedSnapshot,
10361    terminal_ring: &Arc<Mutex<TerminalRing>>,
10362    spawn_events: &SpawnEventFeed,
10363    mut child: SupervisedChild,
10364    drain_timeout: Duration,
10365    final_state: ModuleState,
10366    enabled: Option<bool>,
10367) -> Result<(), SuperviseError> {
10368    let protocol = child.protocol;
10369    update_snapshot(snapshot, Some(module_id), |state| {
10370        state.state = ModuleState::Draining;
10371        state.draining_to_replace = final_state == ModuleState::Restarting;
10372        if let Some(enabled) = enabled {
10373            state.enabled = enabled;
10374        }
10375    })?;
10376
10377    // The wait below is the same budget in every case; what differs is
10378    // whether anything has ASKED the child to stop before it starts. Only a
10379    // forwarding drain that reached the module's registered connection has
10380    // (`module.draining`, then a module GOODBYE). Every other child was told
10381    // nothing: a `protocol: "none"` module, which never registers; a subc
10382    // module spawned moments ago that has not sent HELLO yet; or a stop that
10383    // runs no forwarding drain. Without a signal the budget is only a delay
10384    // in front of SIGKILL -- and the not-yet-registered child is the worst
10385    // case, because it registers into a module that is already draining,
10386    // is never told, and is killed while healthy.
10387    if stop_notice != StopNotice::SentOverConnection {
10388        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10389            info!(
10390                module_id,
10391                pid = child.pid,
10392                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10393                "module has no connection yet; requesting stop by signal"
10394            );
10395        }
10396        request_graceful_stop(module_id, &child);
10397    }
10398
10399    let exit_report = match timeout(drain_timeout, child.wait()).await {
10400        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10401        Ok(Err(source)) => {
10402            fail_after_wait_error(
10403                module_id,
10404                snapshot,
10405                terminal_ring,
10406                spawn_events,
10407                format!("failed to wait during drain: {source}"),
10408            );
10409            return Err(SuperviseError::Wait {
10410                module_id: module_id.to_string(),
10411                source,
10412            });
10413        }
10414        Err(_) => {
10415            // Mirror the sibling arm above: state is already `Draining`, and an
10416            // error propagated from here would strand it there -- a state
10417            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
10418            // `Failed | Stopped`), leaving an operator Restart as the only exit.
10419            // `Failed` before `?` keeps the module operator-visible and
10420            // revivable. Trigger is an ESRCH race (process exits between the
10421            // drain timeout firing and the kill) or a post-kill wait failure
10422            // (issue #34).
10423            //
10424            // Logged because the kill is otherwise visible only as signal 9 in
10425            // the terminal ring, and the budget it follows can be long enough
10426            // that consumers see a stretch of refusals with no stated cause.
10427            warn!(
10428                module_id,
10429                pid = child.pid,
10430                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10431                reason = ?final_state,
10432                ?stop_notice,
10433                "drain budget expired before the module exited; killing it"
10434            );
10435            child.start_kill().map_err(|source| {
10436                fail_after_wait_error(
10437                    module_id,
10438                    snapshot,
10439                    terminal_ring,
10440                    spawn_events,
10441                    format!("failed to kill during drain: {source}"),
10442                );
10443                SuperviseError::Kill {
10444                    module_id: module_id.to_string(),
10445                    source,
10446                }
10447            })?;
10448            let status = child.wait().await.map_err(|source| {
10449                fail_after_wait_error(
10450                    module_id,
10451                    snapshot,
10452                    terminal_ring,
10453                    spawn_events,
10454                    format!("failed to wait after drain kill: {source}"),
10455                );
10456                SuperviseError::Wait {
10457                    module_id: module_id.to_string(),
10458                    source,
10459                }
10460            })?;
10461            classify_reaped_child_exit(snapshot, &child, &status)
10462        }
10463    };
10464
10465    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10466    publish_after_terminal_record(
10467        module_id,
10468        || {
10469            record_terminal_with_detail(
10470                module_id,
10471                terminal_ring,
10472                spawn_events,
10473                &exit_report,
10474                terminal_disposition(final_state),
10475                detail,
10476            )
10477        },
10478        || {
10479            update_snapshot(snapshot, Some(module_id), |state| {
10480                state.state = final_state;
10481                if let Some(enabled) = enabled {
10482                    state.enabled = enabled;
10483                }
10484                clear_current_process_facts(state);
10485                state.last_exit = Some(exit_report.clone());
10486                if exit_report.kind == ExitKind::DeliberateSeverance {
10487                    state.lifetime_restarts += 1;
10488                }
10489            })
10490        },
10491    )?;
10492    child.drain_stderr(module_id).await;
10493
10494    release_dead_registration(registry, forwarding, snapshot, module_id).await
10495}
10496
10497/// Ask a child that nothing else has asked to stop, by signal.
10498///
10499/// A registered subc module is asked over its own connection: the drain sends
10500/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
10501/// module GOODBYE, and the module stops itself. A module that speaks no subc
10502/// wire receives none of that, and neither does a subc module that has not
10503/// registered yet, so for them the drain budget would be pure delay in front of
10504/// a SIGKILL -- and for a process with a store to flush (JetStream is the
10505/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
10506/// into a recovery on the next start.
10507///
10508/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
10509/// rule rather than an optimisation: that module's graceful stop is already
10510/// running by the time its child is drained, and a signal would race it.
10511///
10512/// Best-effort by construction. A child that has already exited is the ordinary
10513/// case rather than an error (the kill lands on a reaped or exiting pid), so a
10514/// failure is logged at debug and the wait-then-kill below still decides the
10515/// outcome.
10516#[cfg(unix)]
10517fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10518    let Some(pid) = child
10519        .id()
10520        .and_then(|pid| i32::try_from(pid).ok())
10521        .and_then(rustix::process::Pid::from_raw)
10522    else {
10523        debug!(
10524            module_id,
10525            "no pid to signal for teardown; falling through to the drain wait"
10526        );
10527        return;
10528    };
10529    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10530        Ok(()) => debug!(
10531            module_id,
10532            "sent SIGTERM to a module nothing else asked to stop"
10533        ),
10534        Err(err) => debug!(
10535            module_id,
10536            error = %err,
10537            "SIGTERM to module failed; the drain wait and kill still apply"
10538        ),
10539    }
10540}
10541
10542/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
10543/// Windows does offer need cooperation this supervisor cannot assume: a console
10544/// control event requires sharing a console with the child, and `WM_CLOSE`
10545/// requires the child to pump a message loop. A supervised server process does
10546/// neither, so there is nothing to send and teardown is the wait followed by the
10547/// kill. Emulating a signal here would mean inventing a stop protocol, which is
10548/// the thing `protocol: "none"` exists to avoid.
10549#[cfg(not(unix))]
10550fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
10551    debug!(
10552        module_id,
10553        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
10554    );
10555}
10556
10557fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
10558    match final_state {
10559        ModuleState::Stopped => TerminalDisposition::Stopped,
10560        ModuleState::Disabled => TerminalDisposition::Disabled,
10561        ModuleState::Restarting => TerminalDisposition::Restarting,
10562        ModuleState::Failed => TerminalDisposition::Failed,
10563        ModuleState::Starting
10564        | ModuleState::Running
10565        | ModuleState::Unresponsive
10566        | ModuleState::Draining => {
10567            unreachable!("terminal exits only finish in terminal or restarting states")
10568        }
10569    }
10570}
10571
10572/// Release a reaped child's registration before allowing another spawn.
10573///
10574/// EOF is not a process-lifetime signal: an inherited socket can stay open
10575/// indefinitely, and serial frame dispatch can be waiting on egress instead of
10576/// reading EOF. After the normal release grace, request connection close (which
10577/// cancels both reads and dispatch), then allow one more release grace for the
10578/// connection guard's forwarding cleanup. Never evict a different connection.
10579async fn release_dead_registration(
10580    registry: &Registry,
10581    forwarding: Option<&ForwardingTable>,
10582    snapshot: &SharedSnapshot,
10583    module_id: &str,
10584) -> Result<(), SuperviseError> {
10585    let result = async {
10586        let registration = registry
10587            .get_module(module_id)
10588            .map_err(SuperviseError::Registry)?;
10589        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
10590            Ok(()) => return Ok(()),
10591            Err(SuperviseError::RegistrationStillActive { .. }) => {}
10592            Err(err) => return Err(err),
10593        }
10594        let pid = lock_snapshot(snapshot)?.reaped_pid;
10595        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
10596            warn!(
10597                module_id,
10598                pid,
10599                connection_id = registration.connection_id.get(),
10600                "reaped module registration outlived release grace; closing dead connection"
10601            );
10602            forwarding.request_connection_close(
10603                registration.connection_id,
10604                CloseReason::new(
10605                    "supervised_process_reaped",
10606                    format!("module '{module_id}' pid {pid} exited"),
10607                ),
10608            );
10609            wait_for_slot_registration_release(
10610                registry,
10611                crate::registry::RegistrationSlot::Connection(registration.connection_id),
10612                REGISTRY_RELEASE_TIMEOUT,
10613            )
10614            .await?;
10615        }
10616        wait_for_registration_release(registry, module_id, Duration::ZERO).await
10617    }
10618    .await;
10619    if let Err(err) = &result {
10620        fail_snapshot(snapshot, Some(module_id), None);
10621        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
10622    }
10623    result
10624}
10625
10626/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
10627/// plain stop or restart waits for before it spawns a replacement.
10628async fn wait_for_registration_release(
10629    registry: &Registry,
10630    module_id: &str,
10631    wait: Duration,
10632) -> Result<(), SuperviseError> {
10633    wait_for_slot_registration_release(
10634        registry,
10635        crate::registry::RegistrationSlot::Active(module_id),
10636        wait,
10637    )
10638    .await
10639}
10640
10641/// Wait for the registration in `slot` to go away.
10642///
10643/// Keyed on the slot rather than the bare module id because a successful swap
10644/// never empties the id's active slot (the promoted candidate is in it), so an
10645/// id-keyed wait for the incumbent's release would always time out. Draining a
10646/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
10647/// incumbent's connection instead.
10648async fn wait_for_slot_registration_release(
10649    registry: &Registry,
10650    slot: crate::registry::RegistrationSlot<'_>,
10651    wait: Duration,
10652) -> Result<(), SuperviseError> {
10653    let deadline = Instant::now() + wait;
10654    let mut release_events = registration_release_events().subscribe();
10655    let still_active = |registration: &crate::registry::ModuleRegistration| {
10656        SuperviseError::RegistrationStillActive {
10657            module_id: registration.manifest.module_id.clone(),
10658            waited: wait,
10659        }
10660    };
10661    loop {
10662        let _observed_generation = *release_events.borrow_and_update();
10663        let Some(registration) = registry
10664            .registration(slot)
10665            .map_err(SuperviseError::Registry)?
10666        else {
10667            return Ok(());
10668        };
10669
10670        let now = Instant::now();
10671        if now >= deadline {
10672            return Err(still_active(&registration));
10673        }
10674
10675        let remaining = deadline.saturating_duration_since(now);
10676        match timeout(remaining, release_events.changed()).await {
10677            Ok(Ok(())) | Ok(Err(_)) => {}
10678            Err(_) => return Err(still_active(&registration)),
10679        }
10680    }
10681}
10682
10683#[cfg(test)]
10684mod slot_registration_wait_tests {
10685    use super::*;
10686    use crate::registry::{ConnectionId, RegistrationSlot};
10687    use subc_protocol::manifest::ModuleManifest;
10688
10689    #[tokio::test]
10690    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
10691        let registry = Arc::new(Registry::default());
10692        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
10693        let runtime = supervisor.runtime_config();
10694        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
10695        let spec = ModuleSpec {
10696            module_id: "enable-stale-registration".to_string(),
10697            program: PathBuf::from("/missing/enable-retry-test"),
10698            args: Vec::new(),
10699            env: Vec::new(),
10700            reserved: false,
10701            reserved_prefixes: Vec::new(),
10702            protocol: ModuleProtocol::Subc,
10703            overlap: Default::default(),
10704        };
10705        let connection = ConnectionId::new(90);
10706        registry
10707            .register_with_control_ops(
10708                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
10709                1,
10710                connection,
10711                Vec::new(),
10712            )
10713            .unwrap();
10714        let mut child = None;
10715        let err = set_child_enabled(
10716            &spec,
10717            &runtime,
10718            &registry,
10719            &supervisor.process_liveness,
10720            &snapshot,
10721            &mut child,
10722            true,
10723        )
10724        .await
10725        .unwrap_err();
10726        assert!(matches!(
10727            err,
10728            SuperviseError::RegistrationStillActive { .. }
10729        ));
10730        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10731        assert!(child.is_none());
10732        registry.deregister_connection(connection).unwrap();
10733        let err = set_child_enabled(
10734            &spec,
10735            &runtime,
10736            &registry,
10737            &supervisor.process_liveness,
10738            &snapshot,
10739            &mut child,
10740            true,
10741        )
10742        .await
10743        .unwrap_err();
10744        assert!(
10745            matches!(err, SuperviseError::Spawn { .. }),
10746            "second enable must attempt a spawn: {err}"
10747        );
10748        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10749    }
10750
10751    const INCUMBENT: u64 = 1;
10752    const CANDIDATE: u64 = 2;
10753
10754    fn swapped_registry() -> Arc<Registry> {
10755        let registry = Arc::new(Registry::default());
10756        let manifest = ModuleManifest::builder("m", "0.1.0").build();
10757        registry
10758            .register_with_control_ops(
10759                manifest.clone(),
10760                1,
10761                ConnectionId::new(INCUMBENT),
10762                Vec::new(),
10763            )
10764            .unwrap();
10765        registry
10766            .register_candidate_with_control_ops(
10767                manifest,
10768                1,
10769                ConnectionId::new(CANDIDATE),
10770                Vec::new(),
10771            )
10772            .unwrap();
10773        registry
10774    }
10775
10776    /// After a promotion the id's active slot is held by the new process, so an
10777    /// id-keyed wait for the incumbent's release can never succeed; the
10778    /// connection-keyed wait completes as soon as the incumbent deregisters.
10779    #[tokio::test]
10780    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
10781        let registry = swapped_registry();
10782        registry.promote_candidate("m").unwrap().unwrap();
10783
10784        assert!(matches!(
10785            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
10786            Err(SuperviseError::RegistrationStillActive { .. })
10787        ));
10788
10789        // Still held while the incumbent's connection has not deregistered.
10790        assert!(matches!(
10791            wait_for_slot_registration_release(
10792                &registry,
10793                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
10794                Duration::from_millis(50),
10795            )
10796            .await,
10797            Err(SuperviseError::RegistrationStillActive { .. })
10798        ));
10799
10800        let releaser = Arc::clone(&registry);
10801        let release = tokio::spawn(async move {
10802            sleep(Duration::from_millis(20)).await;
10803            releaser
10804                .deregister_connection(ConnectionId::new(INCUMBENT))
10805                .unwrap();
10806            notify_registration_release();
10807        });
10808        wait_for_slot_registration_release(
10809            &registry,
10810            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
10811            Duration::from_secs(5),
10812        )
10813        .await
10814        .expect("the incumbent's own registration is released");
10815        release.await.unwrap();
10816        assert!(registry.get_module("m").unwrap().is_some());
10817    }
10818
10819    /// The candidate slot is waited on separately from the active slot: the
10820    /// incumbent's registration neither holds up nor stands in for it.
10821    #[tokio::test]
10822    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
10823        let registry = swapped_registry();
10824        assert!(matches!(
10825            wait_for_slot_registration_release(
10826                &registry,
10827                RegistrationSlot::Candidate("m"),
10828                Duration::from_millis(50),
10829            )
10830            .await,
10831            Err(SuperviseError::RegistrationStillActive { .. })
10832        ));
10833        registry
10834            .deregister_connection(ConnectionId::new(CANDIDATE))
10835            .unwrap();
10836        wait_for_slot_registration_release(
10837            &registry,
10838            RegistrationSlot::Candidate("m"),
10839            Duration::from_millis(50),
10840        )
10841        .await
10842        .expect("a candidate slot with no candidate is released");
10843        assert!(registry
10844            .registration(RegistrationSlot::Active("m"))
10845            .unwrap()
10846            .is_some());
10847    }
10848}
10849
10850fn classify_exit(status: &ExitStatus) -> ExitReport {
10851    ExitReport {
10852        kind: if status.success() {
10853            ExitKind::Clean
10854        } else {
10855            ExitKind::Crash
10856        },
10857        code: status.code(),
10858        signal: exit_signal(status),
10859        at_ms: unix_ms_now(),
10860    }
10861}
10862
10863/// The terminal record for a module whose `wait()` call itself errored (e.g. the
10864/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
10865/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
10866/// disposition still must be `Failed` so the terminal ring is not silently missing
10867/// an entry, matching what `fail_snapshot` records for this same arm.
10868fn wait_error_exit_report() -> ExitReport {
10869    ExitReport {
10870        kind: ExitKind::Crash,
10871        code: None,
10872        signal: None,
10873        at_ms: unix_ms_now(),
10874    }
10875}
10876
10877#[cfg(unix)]
10878fn exit_signal(status: &ExitStatus) -> Option<i32> {
10879    use std::os::unix::process::ExitStatusExt;
10880
10881    status.signal()
10882}
10883
10884#[cfg(not(unix))]
10885fn exit_signal(_status: &ExitStatus) -> Option<i32> {
10886    None
10887}
10888
10889/// Give an operator-touched module its full crash budget back.
10890///
10891/// Named for the counter it used to zero; it now empties the in-window ring,
10892/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
10893/// ledger of what happened survives every operator action.
10894fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
10895    update_snapshot(snapshot, Some(module_id), |state| {
10896        state.clear_crash_restarts();
10897    })
10898}
10899
10900fn set_running(
10901    snapshot: &SharedSnapshot,
10902    child: &SupervisedChild,
10903    module_id: &str,
10904    spawn_events: &SpawnEventFeed,
10905) -> Result<(), SuperviseError> {
10906    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
10907        module_id: Some(module_id.to_string()),
10908    })?;
10909    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
10910    if std::mem::take(&mut state.coalesced_restart_pending) {
10911        let generation = state.spawn_generation;
10912        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
10913    }
10914    state.drain_disposition_detail = None;
10915    state.spawn_failure = None;
10916    state.waiting_for_previous_pid = None;
10917    state.waiting_for_previous_detail = None;
10918    // Every caller of this is a plain spawn, which always uses the primary key;
10919    // a promoted swap candidate sets the flag itself after this returns.
10920    state.in_alternate_slot = false;
10921    state.configuration_updated_since_spawn = false;
10922    state.spawned_protocol = Some(child.protocol);
10923    state.state = ModuleState::Running;
10924    state.enabled = true;
10925    state.process_alive = true;
10926    state.pid = child.id();
10927    #[cfg(windows)]
10928    {
10929        state.pipe_consumed = child
10930            .nonce_delivery
10931            .as_ref()
10932            .map(|delivery| delivery.consumed());
10933    }
10934    #[cfg(target_os = "macos")]
10935    {
10936        state.report_ready = Some(Arc::clone(&child.report_ready));
10937    }
10938    state.spawned_at_ms = Some(child.spawned_at_ms);
10939    state.exec_outcome = Some(Arc::clone(&child.exec.0));
10940    state.spawned_from = Some(child.spawned_from.clone());
10941    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
10942    state.spawned_file_identity = child.spawned_file_identity.clone();
10943    state.process_start_time = child.process_start_time;
10944    Ok(())
10945}
10946
10947fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
10948    state.waiting_for_previous_pid = None;
10949    state.waiting_for_previous_detail = None;
10950    state.process_alive = false;
10951    state.spawned_protocol = None;
10952    state.pid = None;
10953    #[cfg(windows)]
10954    {
10955        state.pipe_consumed = None;
10956    }
10957    #[cfg(target_os = "macos")]
10958    {
10959        state.report_ready = None;
10960    }
10961    state.spawned_at_ms = None;
10962    state.exec_outcome = None;
10963    state.spawned_from = None;
10964    state.launch_spec_digest = None;
10965    state.spawned_file_identity = None;
10966    state.process_start_time = None;
10967    state.deliberate_severance = None;
10968}
10969
10970#[cfg(test)]
10971fn record_deliberate_severance(
10972    snapshot: &SharedSnapshot,
10973    identity: ProcessIdentity,
10974) -> Result<(), SuperviseError> {
10975    update_snapshot(snapshot, None, |state| {
10976        state.deliberate_severance = Some(identity);
10977    })
10978}
10979
10980fn apply_deliberate_severance_marker(
10981    snapshot: &SharedSnapshot,
10982    exited_identity: Option<ProcessIdentity>,
10983    mut exit_report: ExitReport,
10984) -> ExitReport {
10985    let marker = lock_snapshot(snapshot)
10986        .ok()
10987        .and_then(|mut state| state.deliberate_severance.take());
10988    if marker.is_some() && marker == exited_identity {
10989        exit_report.kind = ExitKind::DeliberateSeverance;
10990    }
10991    exit_report
10992}
10993
10994fn classify_reaped_child_exit(
10995    snapshot: &SharedSnapshot,
10996    child: &SupervisedChild,
10997    status: &ExitStatus,
10998) -> ExitReport {
10999    let _ = update_snapshot(snapshot, None, |state| {
11000        state.reaped_pid = Some(child.pid);
11001        state.spawn_failure = child.spawn_failure.clone();
11002    });
11003    apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11004}
11005
11006fn fail_snapshot(
11007    snapshot: &SharedSnapshot,
11008    module_id: Option<&str>,
11009    last_exit: Option<ExitReport>,
11010) {
11011    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11012        state.state = ModuleState::Failed;
11013        clear_current_process_facts(state);
11014        if let Some(last_exit) = last_exit {
11015            state.last_exit = Some(last_exit);
11016        }
11017    }) {
11018        error!(error = %err, "failed to mark supervisor state failed");
11019    }
11020}
11021
11022fn update_snapshot(
11023    snapshot: &SharedSnapshot,
11024    module_id: Option<&str>,
11025    update: impl FnOnce(&mut SupervisorSnapshot),
11026) -> Result<(), SuperviseError> {
11027    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11028        module_id: module_id.map(ToOwned::to_owned),
11029    })?;
11030    update(&mut state);
11031    Ok(())
11032}
11033
11034const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11035
11036fn lock_snapshot_for_control<'a>(
11037    snapshot: &'a SharedSnapshot,
11038    module_id: &str,
11039    caller: &'static str,
11040) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11041    let started_at = Instant::now();
11042    let guard = lock_snapshot(snapshot)?;
11043    let waited = started_at.elapsed();
11044    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11045        warn!(
11046            module_id = %module_id,
11047            waited_ms = waited.as_millis() as u64,
11048            caller = %caller,
11049            "slow snapshot lock"
11050        );
11051    }
11052    Ok(guard)
11053}
11054
11055fn lock_snapshot(
11056    snapshot: &SharedSnapshot,
11057) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11058    snapshot
11059        .lock()
11060        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11061}
11062
11063#[cfg(test)]
11064mod terminal_history_tests {
11065    use std::{
11066        path::PathBuf,
11067        sync::Arc,
11068        time::{Duration, Instant},
11069    };
11070
11071    use tokio::{sync::mpsc, time::sleep};
11072
11073    use super::{
11074        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11075        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11076        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11077        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11078        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11079        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11080        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11081        SupervisorHealthStatus, SupervisorSnapshot,
11082    };
11083    // The supervisor's clock, distinct from the `std::time::Instant` these tests
11084    // use for their own wall-clock deadlines: crash-restart instants must be on
11085    // the same clock the production code stamps them with, which is tokio's (and
11086    // is what `start_paused` tests can move).
11087    use super::Instant as ClockInstant;
11088    use crate::{
11089        registry::Registry,
11090        terminal_ring::{TerminalRing, TerminalRingConfig},
11091    };
11092    use std::sync::Mutex;
11093    use subc_control::TerminalDisposition;
11094
11095    /// See the twin in `control.rs` for why this derives the path from
11096    /// `current_exe()` and why the existence check is here: `--lib` alone does
11097    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
11098    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
11099    pub(super) fn fake_aft_stub_path() -> PathBuf {
11100        let mut path = std::env::current_exe().expect("current_exe available in tests");
11101        path.pop();
11102        path.pop();
11103        path.push(if cfg!(windows) {
11104            "fake-aft-stub.exe"
11105        } else {
11106            "fake-aft-stub"
11107        });
11108        assert!(
11109            path.exists(),
11110            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11111             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11112            path.display()
11113        );
11114        path
11115    }
11116
11117    #[test]
11118    fn reserved_never_spawned_refuses_every_hello() {
11119        // The canary hole: a reserved id whose module has never spawned had NO
11120        // gate entry and admitted anyone -- the reservation protected the nonce
11121        // holder, not the NAME. Now the entry is present with no legitimate
11122        // holder and refuses all comers.
11123        let supervisor = SupervisorHandle::default();
11124        supervisor.apply_identity_configuration(&ModuleSpec {
11125            module_id: "never-spawned".to_string(),
11126            program: PathBuf::from("/usr/bin/false"),
11127            args: Vec::new(),
11128            env: Vec::new(),
11129            reserved: true,
11130            reserved_prefixes: Vec::new(),
11131            protocol: ModuleProtocol::Subc,
11132            overlap: Default::default(),
11133        });
11134        assert!(
11135            supervisor
11136                .reserved_hello_rejection("never-spawned", Some("any-forged-nonce"))
11137                .is_some(),
11138            "forged nonce must refuse on a reserved never-spawned id"
11139        );
11140        assert!(
11141            supervisor
11142                .reserved_hello_rejection("never-spawned", None)
11143                .is_some(),
11144            "absent nonce must refuse on a reserved never-spawned id"
11145        );
11146        // And a real spawn nonce minted later admits exactly that nonce.
11147        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11148        supervisor.apply_identity_configuration(&ModuleSpec {
11149            module_id: "never-spawned".to_string(),
11150            program: PathBuf::from("/usr/bin/false"),
11151            args: Vec::new(),
11152            env: Vec::new(),
11153            reserved: true,
11154            reserved_prefixes: Vec::new(),
11155            protocol: ModuleProtocol::Subc,
11156            overlap: Default::default(),
11157        });
11158        assert!(supervisor
11159            .reserved_hello_rejection("never-spawned", Some("minted"))
11160            .is_none());
11161        assert!(supervisor
11162            .reserved_hello_rejection("never-spawned", Some("forged"))
11163            .is_some());
11164    }
11165
11166    /// Put `count` crash restarts on a snapshot's ring as if they had all just
11167    /// happened, which is what "spent budget" looks like to every reader.
11168    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11169        let now = ClockInstant::now();
11170        for _ in 0..count {
11171            state.crash_restarts.push_back(now);
11172        }
11173    }
11174
11175    /// Age the oldest recorded restart out of `window`, standing in for the hours
11176    /// that would otherwise have to pass. Injecting the instant is the point: a
11177    /// test that slept a real window would take ten minutes and still prove less.
11178    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11179        let aged = state
11180            .crash_restarts
11181            .front()
11182            .expect("a crash restart must be recorded before it can be aged")
11183            .checked_sub(window + Duration::from_secs(1))
11184            .expect("the test clock is far enough from its origin to age an instant");
11185        state.crash_restarts[0] = aged;
11186    }
11187
11188    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11189        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11190        seed_crash_restarts(&mut state, count);
11191        state
11192    }
11193
11194    #[test]
11195    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11196        let policy = RestartPolicy::new(3, Duration::ZERO);
11197        let now = ClockInstant::now();
11198        assert!(daemon_will_restart(
11199            &mut snapshot_with_restarts(true, 2),
11200            &policy,
11201            now
11202        ));
11203        assert!(!daemon_will_restart(
11204            &mut snapshot_with_restarts(true, 3),
11205            &policy,
11206            now
11207        ));
11208        assert!(!daemon_will_restart(
11209            &mut snapshot_with_restarts(false, 0),
11210            &policy,
11211            now
11212        ));
11213    }
11214
11215    #[test]
11216    fn crash_restart_backoff_escalates_with_in_window_count() {
11217        let policy = RestartPolicy::new(4, Duration::from_millis(100))
11218            .with_max_backoff(Duration::from_secs(30));
11219        let now = ClockInstant::now();
11220        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11221        let schedules = (0..4)
11222            .map(|_| {
11223                state
11224                    .next_crash_restart(&policy, now)
11225                    .expect("the test policy allows four crash restarts")
11226            })
11227            .collect::<Vec<_>>();
11228
11229        assert_eq!(
11230            schedules
11231                .iter()
11232                .map(|schedule| schedule.restart_in_window)
11233                .collect::<Vec<_>>(),
11234            vec![0, 1, 2, 3]
11235        );
11236        assert_eq!(
11237            schedules
11238                .iter()
11239                .map(|schedule| schedule.delay)
11240                .collect::<Vec<_>>(),
11241            vec![
11242                Duration::from_millis(100),
11243                Duration::from_secs(1),
11244                Duration::from_secs(10),
11245                Duration::from_secs(30),
11246            ]
11247        );
11248    }
11249
11250    #[test]
11251    fn crash_restart_backoff_resets_after_ring_clear() {
11252        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11253        let now = ClockInstant::now();
11254        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11255        assert_eq!(
11256            state.next_crash_restart(&policy, now).unwrap().delay,
11257            Duration::from_millis(100)
11258        );
11259        assert_eq!(
11260            state.next_crash_restart(&policy, now).unwrap().delay,
11261            Duration::from_secs(1)
11262        );
11263
11264        state.clear_crash_restarts();
11265        let schedule = state
11266            .next_crash_restart(&policy, now)
11267            .expect("a cleared ring must allow another restart");
11268        assert_eq!(schedule.restart_in_window, 0);
11269        assert_eq!(schedule.delay, Duration::from_millis(100));
11270    }
11271
11272    #[test]
11273    fn crash_restart_backoff_ignores_aged_restarts() {
11274        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11275        let now = ClockInstant::now();
11276        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11277        state
11278            .next_crash_restart(&policy, now)
11279            .expect("the first restart is allowed");
11280        state
11281            .next_crash_restart(&policy, now)
11282            .expect("the second restart is allowed");
11283        state.crash_restarts[0] = now
11284            .checked_sub(policy.window + Duration::from_secs(1))
11285            .expect("the fake clock can age a restart past the window");
11286
11287        let schedule = state
11288            .next_crash_restart(&policy, now)
11289            .expect("an aged restart must release its slot");
11290        assert_eq!(schedule.restart_in_window, 1);
11291        assert_eq!(schedule.delay, Duration::from_secs(1));
11292        assert_eq!(state.crash_restarts.len(), 2);
11293    }
11294
11295    /// The budget is a rate: the same three spent restarts refuse a respawn
11296    /// while they are recent and allow one once they have aged past the window.
11297    /// Nothing about the module changed in between, which is the whole point.
11298    #[test]
11299    fn a_budget_spent_before_the_window_no_longer_refuses() {
11300        let policy = RestartPolicy::new(3, Duration::ZERO);
11301        let mut state = snapshot_with_restarts(true, 3);
11302        let now = ClockInstant::now();
11303        assert!(!daemon_will_restart(&mut state, &policy, now));
11304
11305        assert!(daemon_will_restart(
11306            &mut state,
11307            &policy,
11308            now + policy.window + Duration::from_secs(1)
11309        ));
11310        assert!(
11311            state.crash_restarts.is_empty(),
11312            "reading the budget must drop the instants that left the window"
11313        );
11314    }
11315
11316    fn module_with_recovery_snapshot(
11317        state: ModuleState,
11318        enabled: bool,
11319        restart_count: u32,
11320    ) -> SupervisedModule {
11321        let registry = Arc::new(Registry::default());
11322        let supervisor =
11323            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
11324        let runtime = supervisor.runtime_config();
11325        let spec = ModuleSpec {
11326            module_id: "recovery-snapshot".to_string(),
11327            program: fake_aft_stub_path(),
11328            args: Vec::new(),
11329            env: Vec::new(),
11330            reserved: false,
11331            reserved_prefixes: Vec::new(),
11332            protocol: ModuleProtocol::Subc,
11333            overlap: Default::default(),
11334        };
11335        let mut snapshot = SupervisorSnapshot::new(state, enabled);
11336        seed_crash_restarts(&mut snapshot, restart_count);
11337        // These tests read synthetic snapshots. A real child and monitor would
11338        // race those reads by replacing the requested state during startup.
11339        let (commands, _rx) = mpsc::channel(4);
11340        SupervisedModule {
11341            inner: Arc::new(SupervisedModuleInner {
11342                module_id: spec.module_id.clone(),
11343                registry,
11344                snapshot: Arc::new(Mutex::new(snapshot)),
11345                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11346                    spec,
11347                    health: runtime.health,
11348                })),
11349                stderr_ring: runtime.stderr_ring,
11350                terminal_ring: runtime.terminal_ring,
11351                commands,
11352                monitor: Mutex::new(None),
11353                restart_policy: runtime.restart_policy,
11354                effective_drain_timeout: runtime.effective_drain_timeout,
11355                provenance_probe: supervisor.provenance_probe.clone(),
11356            }),
11357        }
11358    }
11359
11360    #[cfg(target_os = "linux")]
11361    #[tokio::test]
11362    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11363        let supervisor =
11364            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11365                .with_cgroup_placement(None);
11366        let result = supervisor.spawn(ModuleSpec {
11367            module_id: "no-cgroup-placement".to_string(),
11368            program: fake_aft_stub_path(),
11369            args: Vec::new(),
11370            env: Vec::new(),
11371            reserved: false,
11372            reserved_prefixes: Vec::new(),
11373            protocol: ModuleProtocol::Subc,
11374            overlap: Default::default(),
11375        });
11376
11377        assert!(
11378            result.is_ok(),
11379            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11380        );
11381    }
11382
11383    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11384    async fn undecided_snapshot_uses_shared_restart_predicate() {
11385        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11386            .will_recover_after_connection_loss()
11387            .unwrap());
11388        assert!(
11389            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11390                .will_recover_after_connection_loss()
11391                .unwrap()
11392        );
11393    }
11394
11395    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11396    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11397        assert!(
11398            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11399                .will_recover_after_connection_loss()
11400                .unwrap()
11401        );
11402    }
11403
11404    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11405    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11406        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11407            .will_recover_after_connection_loss()
11408            .unwrap());
11409        assert!(
11410            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11411                .will_recover_after_connection_loss()
11412                .unwrap()
11413        );
11414    }
11415
11416    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11417    async fn warming_snapshot_is_limited_to_startup_phases() {
11418        for state in [
11419            ModuleState::Starting,
11420            ModuleState::Running,
11421            ModuleState::Restarting,
11422        ] {
11423            assert!(
11424                module_with_recovery_snapshot(state, true, 0)
11425                    .is_warming()
11426                    .unwrap(),
11427                "{state:?} should be warming"
11428            );
11429        }
11430        for state in [
11431            ModuleState::Unresponsive,
11432            ModuleState::Draining,
11433            ModuleState::Stopped,
11434            ModuleState::Failed,
11435            ModuleState::Disabled,
11436        ] {
11437            assert!(
11438                !module_with_recovery_snapshot(state, true, 0)
11439                    .is_warming()
11440                    .unwrap(),
11441                "{state:?} should not be warming"
11442            );
11443        }
11444    }
11445
11446    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11447    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11448        let registry = Arc::new(Registry::default());
11449        let supervisor =
11450            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
11451        let module = supervisor
11452            .spawn(ModuleSpec {
11453                module_id: "terminal-history".to_string(),
11454                program: fake_aft_stub_path(),
11455                args: Vec::new(),
11456                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11457                reserved: false,
11458                reserved_prefixes: Vec::new(),
11459                protocol: ModuleProtocol::Subc,
11460                overlap: Default::default(),
11461            })
11462            .unwrap();
11463
11464        let deadline = Instant::now() + Duration::from_secs(5);
11465        loop {
11466            let history = module.terminal_history();
11467            if history.entries.len() == 2 {
11468                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11469                assert_eq!(history.dropped, 0);
11470                assert_eq!(
11471                    history
11472                        .entries
11473                        .iter()
11474                        .map(|entry| entry.exit_code)
11475                        .collect::<Vec<_>>(),
11476                    vec![Some(23), Some(23)]
11477                );
11478                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11479                return;
11480            }
11481            assert!(
11482                Instant::now() < deadline,
11483                "module did not retain two terminal exits: {history:?}"
11484            );
11485            sleep(Duration::from_millis(10)).await;
11486        }
11487    }
11488
11489    /// A disable issued while a crash respawn is still backing off must preempt
11490    /// that respawn: the operator's stop wins, the disable must not queue behind
11491    /// the backoff, and the module must never come back up afterwards.
11492    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11493    async fn disable_during_crash_backoff_cancels_pending_respawn() {
11494        let backoff = Duration::from_secs(2);
11495        let supervisor = Supervisor::new_for_test(
11496            Arc::new(Registry::default()),
11497            RestartPolicy::new(10, backoff),
11498        );
11499        let module = supervisor
11500            .spawn(ModuleSpec {
11501                module_id: "disable-during-backoff".to_string(),
11502                program: fake_aft_stub_path(),
11503                args: Vec::new(),
11504                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11505                reserved: false,
11506                reserved_prefixes: Vec::new(),
11507                protocol: ModuleProtocol::Subc,
11508                overlap: Default::default(),
11509            })
11510            .unwrap();
11511
11512        // Wait for the first crash to put the module into its backoff window.
11513        let deadline = Instant::now() + Duration::from_secs(5);
11514        loop {
11515            if module.status().unwrap().state == ModuleState::Restarting {
11516                break;
11517            }
11518            assert!(
11519                Instant::now() < deadline,
11520                "module never entered the crash backoff"
11521            );
11522            sleep(Duration::from_millis(10)).await;
11523        }
11524
11525        let started = Instant::now();
11526        module.set_enabled(false).await.unwrap();
11527        let waited = started.elapsed();
11528
11529        assert!(
11530            waited < backoff / 2,
11531            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
11532        );
11533        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
11534
11535        // Outlast the backoff: the respawn it was counting down to must never run.
11536        sleep(backoff + Duration::from_millis(500)).await;
11537        let status = module.status().unwrap();
11538        assert_eq!(status.state, ModuleState::Disabled);
11539        assert_eq!(
11540            status.spawn_generation, 1,
11541            "module respawned after the operator disabled it"
11542        );
11543    }
11544
11545    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
11546    /// the shape of nats-server, the program this rule exists for.
11547    #[cfg(unix)]
11548    fn protocol_none_sigterm_exits_clean_spec(
11549        module_id: &str,
11550        dir: &std::path::Path,
11551    ) -> (ModuleSpec, PathBuf, PathBuf) {
11552        let ready = dir.join("ready");
11553        let marker = dir.join("sigterm");
11554        let spec = ModuleSpec {
11555            module_id: module_id.to_string(),
11556            program: fake_aft_stub_path(),
11557            args: Vec::new(),
11558            env: vec![
11559                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
11560                (
11561                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
11562                    marker.display().to_string(),
11563                ),
11564                (
11565                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
11566                    ready.display().to_string(),
11567                ),
11568            ],
11569            reserved: false,
11570            reserved_prefixes: Vec::new(),
11571            protocol: ModuleProtocol::None,
11572            overlap: Default::default(),
11573        };
11574        (spec, ready, marker)
11575    }
11576
11577    /// Wait for a file the child writes, so a signal is never sent before the
11578    /// child's SIGTERM handler is installed (the default disposition would
11579    /// kill it by signal and the exit would not be clean).
11580    #[cfg(unix)]
11581    async fn wait_for_file(path: &std::path::Path) {
11582        let deadline = Instant::now() + Duration::from_secs(10);
11583        while !path.exists() {
11584            assert!(
11585                Instant::now() < deadline,
11586                "{} never appeared",
11587                path.display()
11588            );
11589            sleep(Duration::from_millis(10)).await;
11590        }
11591    }
11592
11593    /// A protocol-none module that exits 0 because something OUTSIDE the
11594    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
11595    /// the crash-path disposition rather than `stopped`.
11596    #[cfg(unix)]
11597    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11598    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
11599        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
11600        let (spec, ready, marker) =
11601            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
11602        let supervisor = Supervisor::new_for_test(
11603            Arc::new(Registry::default()),
11604            RestartPolicy::new(3, Duration::ZERO),
11605        );
11606        let module = supervisor.spawn(spec).unwrap();
11607        wait_for_file(&ready).await;
11608        // The ready file proves the child installed its SIGTERM handler, not
11609        // that the supervisor has processed the privacy trampoline's exec
11610        // acknowledgement. On macOS status withholds the pid until then.
11611        let deadline = Instant::now() + Duration::from_secs(10);
11612        let first_pid = loop {
11613            let status = module.status().unwrap();
11614            if status.state == ModuleState::Running {
11615                if let Some(pid) = status.pid {
11616                    break pid;
11617                }
11618            }
11619            assert!(
11620                Instant::now() < deadline,
11621                "a running module must report its pid after exec confirmation: {status:?}"
11622            );
11623            sleep(Duration::from_millis(10)).await;
11624        };
11625
11626        rustix::process::kill_process(
11627            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
11628            rustix::process::Signal::TERM,
11629        )
11630        .unwrap();
11631
11632        let deadline = Instant::now() + Duration::from_secs(10);
11633        let respawned = loop {
11634            let status = module.status().unwrap();
11635            if status.state == ModuleState::Running
11636                && status.pid.is_some_and(|pid| pid != first_pid)
11637            {
11638                break status;
11639            }
11640            assert!(
11641                Instant::now() < deadline,
11642                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
11643            );
11644            sleep(Duration::from_millis(10)).await;
11645        };
11646        assert_eq!(respawned.spawn_generation, 2);
11647        assert!(
11648            marker.exists(),
11649            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
11650        );
11651
11652        let history = module.terminal_history();
11653        assert_eq!(history.entries.len(), 1, "{history:?}");
11654        let entry = &history.entries[0];
11655        assert_eq!(entry.exit_code, Some(0));
11656        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
11657        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
11658
11659        module.stop().await.unwrap();
11660    }
11661
11662    /// Repeated unrequested clean exits of a protocol-none module spend the
11663    /// restart budget exactly as crashes do, and the module ends `failed` with
11664    /// the budget named.
11665    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11666    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
11667        let supervisor = Supervisor::new_for_test(
11668            Arc::new(Registry::default()),
11669            RestartPolicy::new(1, Duration::ZERO),
11670        );
11671        let module = supervisor
11672            .spawn(ModuleSpec {
11673                module_id: "none-clean-exit-budget".to_string(),
11674                program: fake_aft_stub_path(),
11675                args: Vec::new(),
11676                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
11677                reserved: false,
11678                reserved_prefixes: Vec::new(),
11679                protocol: ModuleProtocol::None,
11680                overlap: Default::default(),
11681            })
11682            .unwrap();
11683
11684        // Failed follows the terminal write; this deadline only bounds a hang,
11685        // not an assumed duration for the two launches or their exit recording.
11686        let deadline = Instant::now() + Duration::from_secs(10);
11687        loop {
11688            let status = module.status().unwrap();
11689            if status.state == ModuleState::Failed {
11690                break;
11691            }
11692            assert!(
11693                Instant::now() < deadline,
11694                "module never exhausted its budget: {status:?} {:?}",
11695                module.terminal_history()
11696            );
11697            sleep(Duration::from_millis(10)).await;
11698        }
11699        let history = module.terminal_history();
11700        assert_eq!(
11701            history
11702                .entries
11703                .iter()
11704                .map(|entry| (entry.exit_code, entry.disposition.clone()))
11705                .collect::<Vec<_>>(),
11706            vec![
11707                (Some(0), TerminalDisposition::Restarting),
11708                (Some(0), TerminalDisposition::Failed),
11709            ]
11710        );
11711        let detail = history.entries[1]
11712            .disposition_detail
11713            .as_deref()
11714            .expect("a budget failure names the budget");
11715        assert!(detail.contains("max_restarts=1"), "{detail}");
11716        assert_eq!(module.status().unwrap().spawn_generation, 2);
11717    }
11718
11719    #[test]
11720    fn restart_budget_failure_is_published_after_its_terminal_record() {
11721        use std::io::{Read, Write};
11722        let supervisor = Supervisor::new_for_test(
11723            Arc::new(Registry::default()),
11724            RestartPolicy::new(0, Duration::ZERO),
11725        );
11726        let runtime = supervisor.runtime_config();
11727        let spec = ModuleSpec {
11728            protocol: ModuleProtocol::None,
11729            ..windowed_crash_spec("budget-publication-order")
11730        };
11731        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
11732            ModuleState::Running,
11733            true,
11734        )));
11735        lock_snapshot(&snapshot).unwrap().process_alive = true;
11736        let reaped_snapshot = snapshot.clone();
11737        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11738        let history = runtime.terminal_ring.clone();
11739        let reap = std::thread::spawn(move || {
11740            tokio::runtime::Builder::new_current_thread()
11741                .enable_all()
11742                .build()
11743                .unwrap()
11744                .block_on(on_child_exit(
11745                    &spec,
11746                    runtime.restart_policy,
11747                    &supervisor.registry,
11748                    &reaped_snapshot,
11749                    &runtime.terminal_ring,
11750                    &runtime.spawn_events,
11751                    &runtime.child_roster,
11752                    ExitReport {
11753                        kind: ExitKind::Clean,
11754                        code: Some(0),
11755                        signal: None,
11756                        at_ms: 1,
11757                    },
11758                ))
11759        });
11760        let mut reached = [0];
11761        barrier.read_exact(&mut reached).unwrap();
11762        assert_eq!(&reached, b"R");
11763        let (before_state, before_alive) = {
11764            let state = lock_snapshot(&snapshot).unwrap();
11765            (state.state, state.process_alive)
11766        };
11767        let before_history = history.lock().unwrap().snapshot();
11768        barrier.write_all(b"X").unwrap();
11769        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
11770        assert!(before_history.entries.is_empty());
11771        assert_ne!(
11772            before_state,
11773            ModuleState::Failed,
11774            "Failed was visible before its terminal record could be written"
11775        );
11776        assert!(before_alive);
11777        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11778        let history = history.lock().unwrap().snapshot();
11779        assert_eq!(history.entries.len(), 1);
11780        assert_eq!(history.entries[0].exit_code, Some(0));
11781        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11782    }
11783
11784    #[test]
11785    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
11786        use std::io::{Read, Write};
11787
11788        let supervisor = Supervisor::new_for_test(
11789            Arc::new(Registry::default()),
11790            RestartPolicy::new(3, Duration::ZERO),
11791        );
11792        let mut runtime = supervisor.runtime_config();
11793        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
11794        let snapshot = module.inner.snapshot.clone();
11795        lock_snapshot(&snapshot).unwrap().process_alive = true;
11796        runtime.terminal_ring = module.inner.terminal_ring.clone();
11797        let spec = windowed_crash_spec(module.module_id());
11798        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11799        let reaped_snapshot = snapshot.clone();
11800        let reap = std::thread::spawn(move || {
11801            tokio::runtime::Builder::new_current_thread()
11802                .enable_all()
11803                .build()
11804                .unwrap()
11805                .block_on(on_child_exit(
11806                    &spec,
11807                    runtime.restart_policy,
11808                    &supervisor.registry,
11809                    &reaped_snapshot,
11810                    &runtime.terminal_ring,
11811                    &runtime.spawn_events,
11812                    &runtime.child_roster,
11813                    ExitReport {
11814                        kind: ExitKind::Crash,
11815                        code: Some(2),
11816                        signal: None,
11817                        at_ms: 1,
11818                    },
11819                ))
11820        });
11821
11822        // Hold the crash handler between its budget decision and publication.
11823        // Connection cleanup must still see the recovery that this crash chose,
11824        // even when that recovery consumes the last available restart slot.
11825        let mut reached = [0];
11826        barrier.read_exact(&mut reached).unwrap();
11827        assert_eq!(&reached, b"R");
11828        let before_status = module.status().unwrap();
11829        let before_history = module.terminal_history();
11830        let before_recovers = module.will_recover_after_connection_loss().unwrap();
11831        barrier.write_all(b"X").unwrap();
11832        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
11833
11834        assert_eq!(before_status.state, ModuleState::Running);
11835        assert!(before_status.process_alive);
11836        assert_eq!(before_status.restart_count, 3);
11837        assert!(before_history.entries.is_empty());
11838        assert!(
11839            before_recovers,
11840            "connection cleanup would report a terminal close for a scheduled crash restart"
11841        );
11842        let restarting = module.status().unwrap();
11843        assert_eq!(restarting.state, ModuleState::Restarting);
11844        assert_eq!(restarting.restart_count, 3);
11845        assert!(module.will_recover_after_connection_loss().unwrap());
11846        let history = module.terminal_history();
11847        assert_eq!(history.entries.len(), 1);
11848        assert_eq!(
11849            history.entries[0].disposition,
11850            TerminalDisposition::Restarting
11851        );
11852
11853        // Once the replacement is running, its next crash must be judged
11854        // against the exhausted budget rather than the previous recovery.
11855        update_snapshot(&snapshot, Some(module.module_id()), |state| {
11856            state.state = ModuleState::Running;
11857            state.respawn_pending = false;
11858        })
11859        .unwrap();
11860        assert!(!module.will_recover_after_connection_loss().unwrap());
11861        update_snapshot(&snapshot, Some(module.module_id()), |state| {
11862            state.enabled = false;
11863            state.respawn_pending = true;
11864        })
11865        .unwrap();
11866        assert!(!module.will_recover_after_connection_loss().unwrap());
11867    }
11868
11869    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
11870        use std::net::{TcpListener, TcpStream};
11871        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
11872        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
11873        let (barrier, _) = listener.accept().unwrap();
11874        for stream in [&writer, &barrier] {
11875            stream
11876                .set_read_timeout(Some(Duration::from_secs(10)))
11877                .unwrap();
11878            stream
11879                .set_write_timeout(Some(Duration::from_secs(10)))
11880                .unwrap();
11881        }
11882        *events.1.lock().unwrap() = Some(writer);
11883        barrier
11884    }
11885
11886    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
11887        use std::io::{Read, Write};
11888        let barrier = events.1.lock().unwrap().take();
11889        if let Some(mut barrier) = barrier {
11890            barrier.write_all(b"R").unwrap();
11891            let mut release = [0];
11892            barrier.read_exact(&mut release).unwrap();
11893            assert_eq!(&release, b"X");
11894        }
11895    }
11896
11897    #[test]
11898    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
11899        use std::io::{Read, Write};
11900
11901        let supervisor = Supervisor::new_for_test(
11902            Arc::new(Registry::default()),
11903            RestartPolicy::new(0, Duration::ZERO),
11904        );
11905        let mut runtime = supervisor.runtime_config();
11906        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
11907        let snapshot = module.inner.snapshot.clone();
11908        runtime.terminal_ring = module.inner.terminal_ring.clone();
11909        let spec = ModuleSpec {
11910            module_id: module.module_id().into(),
11911            program: fake_aft_stub_path(),
11912            protocol: ModuleProtocol::None,
11913            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
11914            ..windowed_crash_spec(module.module_id())
11915        };
11916        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11917        let reap = std::thread::spawn(move || {
11918            tokio::runtime::Builder::new_current_thread()
11919                .enable_all()
11920                .build()
11921                .unwrap()
11922                .block_on(async {
11923                    let mut child =
11924                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
11925                    health_restart_child(
11926                        &spec,
11927                        &runtime,
11928                        &supervisor.registry,
11929                        &supervisor.process_liveness,
11930                        &snapshot,
11931                        &mut child,
11932                        SupervisorHealthStatus::Failing,
11933                        Some("health probe did not answer"),
11934                        1,
11935                    )
11936                    .await
11937                    .unwrap();
11938                    assert!(child.is_none());
11939                });
11940        });
11941        // The loopback barrier acknowledges entry into terminal recording by
11942        // health_restart_child's drain, then waits for the test's release byte.
11943        // Status and history are observed while that writer is held, not after
11944        // an elapsed-time guess about when the health action has completed.
11945        let mut reached = [0];
11946        barrier.read_exact(&mut reached).unwrap();
11947        assert_eq!(&reached, b"R");
11948        let before_status = module.status().unwrap();
11949        let before_history = module.terminal_history();
11950        barrier.write_all(b"X").unwrap();
11951        reap.join().unwrap();
11952        assert!(before_history.entries.is_empty());
11953        assert_ne!(
11954            before_status.state,
11955            ModuleState::Failed,
11956            "Failed was visible before its terminal record could be written"
11957        );
11958        assert!(
11959            before_status.process_alive,
11960            "process death was visible before its terminal record could be written"
11961        );
11962        let failed = module.status().unwrap();
11963        assert_eq!(failed.state, ModuleState::Failed);
11964        assert!(!failed.process_alive);
11965        let history = module.terminal_history();
11966        assert_eq!(history.entries.len(), 1);
11967        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11968        assert_eq!(
11969            history.entries[0].disposition_detail.as_deref(),
11970            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
11971        );
11972    }
11973
11974    #[test]
11975    fn observed_exits_publish_process_death_after_terminal_history() {
11976        use std::io::{Read, Write};
11977        for (kind, enabled, shutdown, wait_error, expected_state) in [
11978            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
11979            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
11980            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
11981            (
11982                ExitKind::DeliberateSeverance,
11983                true,
11984                false,
11985                false,
11986                ModuleState::Restarting,
11987            ),
11988            #[cfg(unix)]
11989            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
11990            (ExitKind::Crash, true, false, true, ModuleState::Failed),
11991        ] {
11992            let supervisor = Supervisor::new_for_test(
11993                Arc::new(Registry::default()),
11994                RestartPolicy::new(1, Duration::ZERO),
11995            );
11996            let runtime = supervisor.runtime_config();
11997            #[cfg(unix)]
11998            if shutdown {
11999                runtime.child_roster.close();
12000            }
12001            #[cfg(not(unix))]
12002            let _ = shutdown;
12003            let spec = windowed_crash_spec("exit-publication-order");
12004            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12005                ModuleState::Running,
12006                enabled,
12007            )));
12008            lock_snapshot(&snapshot).unwrap().process_alive = true;
12009            let reaped_snapshot = snapshot.clone();
12010            let history = runtime.terminal_ring.clone();
12011            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12012            let reap = std::thread::spawn(move || {
12013                if wait_error {
12014                    super::fail_after_wait_error(
12015                        &spec.module_id,
12016                        &reaped_snapshot,
12017                        &runtime.terminal_ring,
12018                        &runtime.spawn_events,
12019                        "wait failed".into(),
12020                    );
12021                } else {
12022                    tokio::runtime::Builder::new_current_thread()
12023                        .enable_all()
12024                        .build()
12025                        .unwrap()
12026                        .block_on(on_child_exit(
12027                            &spec,
12028                            runtime.restart_policy,
12029                            &supervisor.registry,
12030                            &reaped_snapshot,
12031                            &runtime.terminal_ring,
12032                            &runtime.spawn_events,
12033                            &runtime.child_roster,
12034                            ExitReport {
12035                                kind,
12036                                code: Some(0),
12037                                signal: None,
12038                                at_ms: 1,
12039                            },
12040                        ));
12041                }
12042            });
12043            let mut reached = [0];
12044            barrier.read_exact(&mut reached).unwrap();
12045            assert_eq!(&reached, b"R");
12046            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12047            let before_history = history.lock().unwrap().snapshot();
12048            barrier.write_all(b"X").unwrap();
12049            reap.join().unwrap();
12050            assert!(
12051                before_alive,
12052                "{kind:?}: death was published before recording"
12053            );
12054            assert!(before_history.entries.is_empty());
12055            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12056            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12057            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12058        }
12059    }
12060
12061    #[tokio::test]
12062    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12063        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12064        let path = dir.join("terminals.jsonl");
12065        std::fs::create_dir(&path).unwrap();
12066        let supervisor = Supervisor::new_for_test(
12067            Arc::new(Registry::default()),
12068            RestartPolicy::new(0, Duration::ZERO),
12069        )
12070        .with_terminal_journal(path, "health-budget-journal-failure".into());
12071        let runtime = supervisor.runtime_config();
12072        let spec = ModuleSpec {
12073            program: fake_aft_stub_path(),
12074            protocol: ModuleProtocol::None,
12075            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12076            ..windowed_crash_spec("health-budget-journal-failure")
12077        };
12078        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12079        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12080        health_restart_child(
12081            &spec,
12082            &runtime,
12083            &supervisor.registry,
12084            &supervisor.process_liveness,
12085            &snapshot,
12086            &mut child,
12087            SupervisorHealthStatus::Failing,
12088            None,
12089            1,
12090        )
12091        .await
12092        .unwrap();
12093        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12094        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12095        let history = runtime
12096            .terminal_ring
12097            .lock()
12098            .unwrap()
12099            .durable_history(&spec.module_id);
12100        assert!(history.journal_write_failures > 0);
12101        assert_eq!(history.entries.len(), 1);
12102        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12103    }
12104
12105    #[test]
12106    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12107        let supervisor = Supervisor::new_for_test(
12108            Arc::new(Registry::default()),
12109            RestartPolicy::new(0, Duration::ZERO),
12110        );
12111        let runtime = supervisor.runtime_config();
12112        let spec = ModuleSpec {
12113            program: fake_aft_stub_path(),
12114            protocol: ModuleProtocol::None,
12115            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12116            ..windowed_crash_spec("health-budget-recording-panic")
12117        };
12118        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12119        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12120            tokio::runtime::Builder::new_current_thread()
12121                .enable_all()
12122                .build()
12123                .unwrap()
12124                .block_on(async {
12125                    let mut child =
12126                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12127                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12128                    health_restart_child(
12129                        &spec,
12130                        &runtime,
12131                        &supervisor.registry,
12132                        &supervisor.process_liveness,
12133                        &snapshot,
12134                        &mut child,
12135                        SupervisorHealthStatus::Failing,
12136                        None,
12137                        1,
12138                    )
12139                    .await
12140                    .unwrap();
12141                });
12142        }));
12143        let panic = result.expect_err("recording must still unwind");
12144        assert_eq!(
12145            panic.downcast_ref::<String>().map(String::as_str),
12146            Some("spawn event sequence exhausted")
12147        );
12148        let state = lock_snapshot(&snapshot).unwrap();
12149        assert_eq!(state.state, ModuleState::Failed);
12150        assert!(!state.process_alive);
12151        assert!(state.last_exit.is_some());
12152    }
12153
12154    #[tokio::test]
12155    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12156        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12157        let path = dir.join("terminals.jsonl");
12158        std::fs::create_dir(&path).unwrap();
12159        let supervisor = Supervisor::new_for_test(
12160            Arc::new(Registry::default()),
12161            RestartPolicy::new(0, Duration::ZERO),
12162        )
12163        .with_terminal_journal(path, "budget-journal-failure".into());
12164        let runtime = supervisor.runtime_config();
12165        let spec = windowed_crash_spec("budget-journal-failure");
12166        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12167        assert!(matches!(
12168            on_child_exit(
12169                &spec,
12170                runtime.restart_policy,
12171                &supervisor.registry,
12172                &snapshot,
12173                &runtime.terminal_ring,
12174                &runtime.spawn_events,
12175                &runtime.child_roster,
12176                crash_exit_report(1),
12177            )
12178            .await,
12179            NextAction::Stop { .. }
12180        ));
12181        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12182        let history = runtime
12183            .terminal_ring
12184            .lock()
12185            .unwrap()
12186            .durable_history(&spec.module_id);
12187        assert!(history.journal_write_failures > 0);
12188        assert_eq!(history.entries.len(), 1);
12189        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12190    }
12191
12192    #[test]
12193    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12194        let supervisor = Supervisor::new_for_test(
12195            Arc::new(Registry::default()),
12196            RestartPolicy::new(0, Duration::ZERO),
12197        );
12198        let runtime = supervisor.runtime_config();
12199        let spec = windowed_crash_spec("budget-recording-panic");
12200        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12201        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12202        // Exhausting the event sequence makes emit_exited panic before the
12203        // terminal write, exercising publication on the recording unwind.
12204        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12205        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12206            tokio::runtime::Builder::new_current_thread()
12207                .enable_all()
12208                .build()
12209                .unwrap()
12210                .block_on(on_child_exit(
12211                    &spec,
12212                    runtime.restart_policy,
12213                    &supervisor.registry,
12214                    &snapshot,
12215                    &runtime.terminal_ring,
12216                    &runtime.spawn_events,
12217                    &runtime.child_roster,
12218                    crash_exit_report(1),
12219                ))
12220        }));
12221        let panic = result.err().expect("recording must still unwind");
12222        assert_eq!(
12223            panic.downcast_ref::<String>().map(String::as_str),
12224            Some("spawn event sequence exhausted")
12225        );
12226        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12227    }
12228
12229    /// A stop the supervisor itself requests still stops a protocol-none
12230    /// module, even though the child answers the SIGTERM with exit 0.
12231    #[cfg(unix)]
12232    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12233    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12234        for disable in [false, true] {
12235            let label = if disable {
12236                "none-requested-disable"
12237            } else {
12238                "none-requested-stop"
12239            };
12240            let dir = cortexkit_test_support::ScratchDir::new(label);
12241            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12242            let supervisor = Supervisor::new_for_test(
12243                Arc::new(Registry::default()),
12244                RestartPolicy::new(3, Duration::ZERO),
12245            );
12246            let module = supervisor.spawn(spec).unwrap();
12247            wait_for_file(&ready).await;
12248
12249            if disable {
12250                module.set_enabled(false).await.unwrap();
12251            } else {
12252                module.stop().await.unwrap();
12253            }
12254            assert!(
12255                marker.exists(),
12256                "{label}: the child must have left through its SIGTERM handler with exit 0"
12257            );
12258
12259            // Long enough for a zero-backoff respawn to have happened if the
12260            // exit had been treated as a crash.
12261            sleep(Duration::from_millis(500)).await;
12262            let status = module.status().unwrap();
12263            let expected = if disable {
12264                ModuleState::Disabled
12265            } else {
12266                ModuleState::Stopped
12267            };
12268            assert_eq!(status.state, expected, "{label}");
12269            assert_eq!(
12270                status.spawn_generation, 1,
12271                "{label}: respawned after a requested stop"
12272            );
12273            let history = module.terminal_history();
12274            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12275            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12276            assert_ne!(
12277                history.entries[0].disposition,
12278                TerminalDisposition::Restarting,
12279                "{label}"
12280            );
12281        }
12282    }
12283
12284    /// A subc-wire module that exits 0 on its own is still a stop: the
12285    /// protocol-none rule must not reach it.
12286    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12287    async fn subc_wire_clean_exit_is_still_a_stop() {
12288        let supervisor = Supervisor::new_for_test(
12289            Arc::new(Registry::default()),
12290            RestartPolicy::new(3, Duration::ZERO),
12291        );
12292        let module = supervisor
12293            .spawn(ModuleSpec {
12294                module_id: "wire-clean-exit".to_string(),
12295                program: fake_aft_stub_path(),
12296                args: Vec::new(),
12297                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12298                reserved: false,
12299                reserved_prefixes: Vec::new(),
12300                protocol: ModuleProtocol::Subc,
12301                overlap: Default::default(),
12302            })
12303            .unwrap();
12304
12305        let deadline = Instant::now() + Duration::from_secs(10);
12306        while module.terminal_history().entries.is_empty() {
12307            assert!(Instant::now() < deadline, "module never exited");
12308            sleep(Duration::from_millis(10)).await;
12309        }
12310        // Long enough for a zero-backoff respawn to have happened.
12311        sleep(Duration::from_millis(500)).await;
12312        let status = module.status().unwrap();
12313        assert_eq!(status.state, ModuleState::Stopped);
12314        assert_eq!(status.spawn_generation, 1);
12315        let history = module.terminal_history();
12316        assert_eq!(history.entries.len(), 1, "{history:?}");
12317        assert_eq!(history.entries[0].exit_code, Some(0));
12318        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12319    }
12320
12321    #[cfg(unix)]
12322    #[tokio::test]
12323    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12324        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12325        let record = dir.join("live-children.json");
12326        let supervisor = Supervisor::new_for_test(
12327            Arc::new(Registry::default()),
12328            RestartPolicy::new(0, Duration::ZERO),
12329        );
12330        let mut runtime = supervisor.runtime_config();
12331        runtime.child_roster.record_to(record.clone());
12332        let gate = Arc::new(super::ReloadExitRecordGate::default());
12333        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12334        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12335        let spec = ModuleSpec {
12336            module_id: "reload-exit-roster".into(),
12337            program: fake_aft_stub_path(),
12338            args: Vec::new(),
12339            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12340            reserved: false,
12341            reserved_prefixes: Vec::new(),
12342            protocol: ModuleProtocol::Subc,
12343            overlap: Default::default(),
12344        };
12345        let mut child = None;
12346        let reload = super::finish_reload_child(
12347            &spec,
12348            &runtime,
12349            &supervisor.registry,
12350            &supervisor.process_liveness,
12351            &snapshot,
12352            &mut child,
12353        );
12354        tokio::pin!(reload);
12355        tokio::select! {
12356            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12357            _ = gate.reached.notified() => {}
12358        }
12359        assert!(runtime
12360            .terminal_ring
12361            .lock()
12362            .unwrap()
12363            .snapshot()
12364            .entries
12365            .is_empty());
12366        assert_eq!(
12367            crate::live_children::read_record(&record).unwrap().len(),
12368            1,
12369            "shutdown must still wait for the reaped child until its terminal record exists"
12370        );
12371        runtime.child_roster.close();
12372        gate.resume.notify_one();
12373        assert!(reload.await.is_err());
12374        assert!(crate::live_children::read_record(&record)
12375            .unwrap()
12376            .is_empty());
12377        let history = runtime.terminal_ring.lock().unwrap().snapshot();
12378        assert_eq!(history.entries.len(), 1);
12379        assert_eq!(
12380            history.entries[0].disposition,
12381            TerminalDisposition::DaemonShutdown
12382        );
12383    }
12384
12385    /// Each restart-producing arm has its own state transition. Keeping their
12386    /// lifetime count assertions adjacent prevents a later new arm from silently
12387    /// spending budget without recording the historical restart.
12388    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12389    async fn every_restart_increment_path_advances_lifetime_count() {
12390        let supervisor = Supervisor::new_for_test(
12391            Arc::new(Registry::default()),
12392            RestartPolicy::new(1, Duration::ZERO),
12393        );
12394        let runtime = supervisor.runtime_config();
12395        let spec = ModuleSpec {
12396            module_id: "lifetime-increment-path".to_string(),
12397            program: PathBuf::from("/unused/lifetime-increment-path"),
12398            args: Vec::new(),
12399            env: Vec::new(),
12400            reserved: false,
12401            reserved_prefixes: Vec::new(),
12402            protocol: ModuleProtocol::Subc,
12403            overlap: Default::default(),
12404        };
12405
12406        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12407        assert!(matches!(
12408            on_child_exit(
12409                &spec,
12410                runtime.restart_policy,
12411                &supervisor.registry,
12412                &crash_snapshot,
12413                &runtime.terminal_ring,
12414                &runtime.spawn_events,
12415                &runtime.child_roster,
12416                ExitReport {
12417                    kind: ExitKind::Crash,
12418                    code: Some(1),
12419                    signal: None,
12420                    at_ms: 1,
12421                },
12422            )
12423            .await,
12424            NextAction::Restart { schedule: _ }
12425        ));
12426        let (crash_restarts, crash_lifetime) = {
12427            let state = lock_snapshot(&crash_snapshot).unwrap();
12428            (state.crash_restarts.len(), state.lifetime_restarts)
12429        };
12430        assert_eq!(crash_restarts, 1);
12431        assert_eq!(crash_lifetime, 1);
12432
12433        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12434        let mut health_child = None;
12435        assert!(matches!(
12436            health_restart_child(
12437                &spec,
12438                &runtime,
12439                &supervisor.registry,
12440                &supervisor.process_liveness,
12441                &health_snapshot,
12442                &mut health_child,
12443                SupervisorHealthStatus::Failing,
12444                None,
12445                2,
12446            )
12447            .await,
12448            Ok(())
12449        ));
12450        assert!(health_child.is_none());
12451        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12452        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12453        let (health_restarts, health_lifetime) = {
12454            let state = lock_snapshot(&health_snapshot).unwrap();
12455            (state.crash_restarts.len(), state.lifetime_restarts)
12456        };
12457        assert_eq!(health_restarts, 1);
12458        assert_eq!(health_lifetime, 1);
12459
12460        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12461        let mut reload_child = None;
12462        assert!(matches!(
12463            handle_reload_spawn_failure(
12464                &spec,
12465                &runtime,
12466                &supervisor.process_liveness,
12467                &reload_snapshot,
12468                &mut reload_child,
12469                "forced reload spawn failure".to_string(),
12470            )
12471            .await,
12472            Err(SuperviseError::ReloadFailed { .. })
12473        ));
12474        let (reload_restarts, reload_lifetime) = {
12475            let state = lock_snapshot(&reload_snapshot).unwrap();
12476            (state.crash_restarts.len(), state.lifetime_restarts)
12477        };
12478        assert_eq!(reload_restarts, 1);
12479        assert_eq!(reload_lifetime, 1);
12480    }
12481
12482    #[tokio::test]
12483    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12484        let supervisor = Supervisor::new_for_test(
12485            Arc::new(Registry::default()),
12486            RestartPolicy::new(3, Duration::ZERO),
12487        );
12488        let runtime = supervisor.runtime_config();
12489        let spec = ModuleSpec {
12490            module_id: "deliberately-severed".to_string(),
12491            program: PathBuf::from("/unused/deliberately-severed"),
12492            args: Vec::new(),
12493            env: Vec::new(),
12494            reserved: false,
12495            reserved_prefixes: Vec::new(),
12496            protocol: ModuleProtocol::Subc,
12497            overlap: Default::default(),
12498        };
12499        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12500        let process = ProcessIdentity {
12501            pid: 41,
12502            start_time: 101,
12503        };
12504        record_deliberate_severance(&snapshot, process).unwrap();
12505        let exit_report = apply_deliberate_severance_marker(
12506            &snapshot,
12507            Some(process),
12508            ExitReport {
12509                kind: ExitKind::Crash,
12510                code: Some(1),
12511                signal: None,
12512                at_ms: 1,
12513            },
12514        );
12515        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12516
12517        assert!(matches!(
12518            on_child_exit(
12519                &spec,
12520                runtime.restart_policy,
12521                &supervisor.registry,
12522                &snapshot,
12523                &runtime.terminal_ring,
12524                &runtime.spawn_events,
12525                &runtime.child_roster,
12526                exit_report,
12527            )
12528            .await,
12529            NextAction::Restart { schedule: _ }
12530        ));
12531        let state = lock_snapshot(&snapshot).unwrap();
12532        assert_eq!(state.lifetime_restarts, 1);
12533        assert_eq!(state.crash_restarts.len(), 0);
12534    }
12535
12536    #[tokio::test]
12537    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
12538        let supervisor = Supervisor::new_for_test(
12539            Arc::new(Registry::default()),
12540            RestartPolicy::new(3, Duration::ZERO),
12541        );
12542        let runtime = supervisor.runtime_config();
12543        let spec = ModuleSpec {
12544            module_id: "genuine-crash".to_string(),
12545            program: PathBuf::from("/unused/genuine-crash"),
12546            args: Vec::new(),
12547            env: Vec::new(),
12548            reserved: false,
12549            reserved_prefixes: Vec::new(),
12550            protocol: ModuleProtocol::Subc,
12551            overlap: Default::default(),
12552        };
12553        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12554
12555        assert!(matches!(
12556            on_child_exit(
12557                &spec,
12558                runtime.restart_policy,
12559                &supervisor.registry,
12560                &snapshot,
12561                &runtime.terminal_ring,
12562                &runtime.spawn_events,
12563                &runtime.child_roster,
12564                ExitReport {
12565                    kind: ExitKind::Crash,
12566                    code: Some(1),
12567                    signal: None,
12568                    at_ms: 1,
12569                },
12570            )
12571            .await,
12572            NextAction::Restart { schedule: _ }
12573        ));
12574        let state = lock_snapshot(&snapshot).unwrap();
12575        assert_eq!(state.lifetime_restarts, 1);
12576        assert_eq!(state.crash_restarts.len(), 1);
12577    }
12578
12579    fn crash_exit_report(at_ms: u64) -> ExitReport {
12580        ExitReport {
12581            kind: ExitKind::Crash,
12582            code: Some(1),
12583            signal: None,
12584            at_ms,
12585        }
12586    }
12587
12588    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
12589        ModuleSpec {
12590            module_id: module_id.to_string(),
12591            program: PathBuf::from("/unused").join(module_id),
12592            args: Vec::new(),
12593            env: Vec::new(),
12594            reserved: false,
12595            reserved_prefixes: Vec::new(),
12596            protocol: ModuleProtocol::Subc,
12597            overlap: Default::default(),
12598        }
12599    }
12600
12601    /// A real crash loop still stops. Three crashes with nothing aging out spend
12602    /// a budget of two and the third respawn is refused, and both surfaces an
12603    /// operator has -- the log line and the retained terminal record -- name the
12604    /// window rather than only the cap, because `max_restarts=2` alone is what
12605    /// this budget used to mean.
12606    #[tokio::test]
12607    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
12608        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
12609        let supervisor = Supervisor::new_for_test(
12610            Arc::new(Registry::default()),
12611            RestartPolicy::new(2, Duration::ZERO),
12612        );
12613        let runtime = supervisor.runtime_config();
12614        let spec = windowed_crash_spec("crash-loop-in-window");
12615        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12616
12617        for attempt in 1..=2 {
12618            assert!(
12619                matches!(
12620                    on_child_exit(
12621                        &spec,
12622                        runtime.restart_policy,
12623                        &supervisor.registry,
12624                        &snapshot,
12625                        &runtime.terminal_ring,
12626                        &runtime.spawn_events,
12627                        &runtime.child_roster,
12628                        crash_exit_report(attempt),
12629                    )
12630                    .await,
12631                    NextAction::Restart { schedule: _ }
12632                ),
12633                "crash {attempt} is inside the budget and must respawn"
12634            );
12635        }
12636
12637        assert!(matches!(
12638            on_child_exit(
12639                &spec,
12640                runtime.restart_policy,
12641                &supervisor.registry,
12642                &snapshot,
12643                &runtime.terminal_ring,
12644                &runtime.spawn_events,
12645                &runtime.child_roster,
12646                crash_exit_report(3),
12647            )
12648            .await,
12649            NextAction::Stop { .. }
12650        ));
12651
12652        {
12653            let state = lock_snapshot(&snapshot).unwrap();
12654            assert_eq!(state.state, ModuleState::Failed);
12655            assert_eq!(state.crash_restarts.len(), 2);
12656            assert_eq!(state.lifetime_restarts, 2);
12657        }
12658
12659        let history = runtime
12660            .terminal_ring
12661            .lock()
12662            .expect("terminal ring is not poisoned")
12663            .snapshot();
12664        let last = history
12665            .entries
12666            .last()
12667            .expect("the refused crash is retained");
12668        assert_eq!(last.disposition, TerminalDisposition::Failed);
12669        assert_eq!(
12670            last.disposition_detail.as_deref(),
12671            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
12672        );
12673
12674        let captured = crate::router::test_log::captured_logs(&logs);
12675        assert!(
12676            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
12677            "the stop must be logged with its window: {captured}"
12678        );
12679    }
12680
12681    /// The rate, stated as a test: three crashes where the first has aged past
12682    /// the window are two crashes as far as the budget is concerned, so the
12683    /// third respawn is allowed and the ring holds only the two recent ones.
12684    ///
12685    /// This is the case a lifetime counter got wrong -- and the case the daemon
12686    /// now hits routinely, since a module exits non-zero every time its
12687    /// connection to the daemon drops.
12688    #[tokio::test]
12689    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
12690        let supervisor = Supervisor::new_for_test(
12691            Arc::new(Registry::default()),
12692            RestartPolicy::new(2, Duration::ZERO),
12693        );
12694        let runtime = supervisor.runtime_config();
12695        let spec = windowed_crash_spec("crash-across-windows");
12696        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12697
12698        for attempt in 1..=2 {
12699            assert!(matches!(
12700                on_child_exit(
12701                    &spec,
12702                    runtime.restart_policy,
12703                    &supervisor.registry,
12704                    &snapshot,
12705                    &runtime.terminal_ring,
12706                    &runtime.spawn_events,
12707                    &runtime.child_roster,
12708                    crash_exit_report(attempt),
12709                )
12710                .await,
12711                NextAction::Restart { schedule: _ }
12712            ));
12713        }
12714
12715        // The oldest crash moves out of the window; nothing else about the
12716        // module changes.
12717        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12718            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
12719        })
12720        .unwrap();
12721
12722        assert!(
12723            matches!(
12724                on_child_exit(
12725                    &spec,
12726                    runtime.restart_policy,
12727                    &supervisor.registry,
12728                    &snapshot,
12729                    &runtime.terminal_ring,
12730                    &runtime.spawn_events,
12731                    &runtime.child_roster,
12732                    crash_exit_report(3),
12733                )
12734                .await,
12735                NextAction::Restart { schedule: _ }
12736            ),
12737            "a crash older than the window must not hold a budget slot"
12738        );
12739
12740        let state = lock_snapshot(&snapshot).unwrap();
12741        assert_eq!(state.state, ModuleState::Restarting);
12742        assert_eq!(
12743            state.crash_restarts.len(),
12744            2,
12745            "the aged instant is dropped and the new one takes its place"
12746        );
12747        assert_eq!(
12748            state.lifetime_restarts, 3,
12749            "the ledger counts every restart, including the ones the window forgot"
12750        );
12751    }
12752
12753    /// An operator restart hands the budget back whole, and the ledger keeps
12754    /// counting. Those are different questions -- "how close is this module to
12755    /// being stopped" and "how many times has it been replaced" -- and the
12756    /// operator action answers only the first.
12757    #[tokio::test]
12758    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
12759        let supervisor = Supervisor::new_for_test(
12760            Arc::new(Registry::default()),
12761            RestartPolicy::new(2, Duration::ZERO),
12762        );
12763        let runtime = supervisor.runtime_config();
12764        let spec = windowed_crash_spec("operator-cleared-budget");
12765        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12766
12767        for attempt in 1..=2 {
12768            assert!(matches!(
12769                on_child_exit(
12770                    &spec,
12771                    runtime.restart_policy,
12772                    &supervisor.registry,
12773                    &snapshot,
12774                    &runtime.terminal_ring,
12775                    &runtime.spawn_events,
12776                    &runtime.child_roster,
12777                    crash_exit_report(attempt),
12778                )
12779                .await,
12780                NextAction::Restart { schedule: _ }
12781            ));
12782        }
12783
12784        reset_restart_count(&snapshot, &spec.module_id).unwrap();
12785        {
12786            let state = lock_snapshot(&snapshot).unwrap();
12787            assert!(
12788                state.crash_restarts.is_empty(),
12789                "an operator restart returns the full budget"
12790            );
12791            assert_eq!(
12792                state.lifetime_restarts, 2,
12793                "clearing the budget must not unmake the crashes"
12794            );
12795        }
12796
12797        assert!(
12798            matches!(
12799                on_child_exit(
12800                    &spec,
12801                    runtime.restart_policy,
12802                    &supervisor.registry,
12803                    &snapshot,
12804                    &runtime.terminal_ring,
12805                    &runtime.spawn_events,
12806                    &runtime.child_roster,
12807                    crash_exit_report(3),
12808                )
12809                .await,
12810                NextAction::Restart { schedule: _ }
12811            ),
12812            "the cleared budget must be spendable again"
12813        );
12814        let state = lock_snapshot(&snapshot).unwrap();
12815        assert_eq!(state.crash_restarts.len(), 1);
12816        assert_eq!(state.lifetime_restarts, 3);
12817    }
12818
12819    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12820    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
12821        let severed = ProcessIdentity {
12822            pid: 41,
12823            start_time: 101,
12824        };
12825        let successor = ProcessIdentity {
12826            pid: 41,
12827            start_time: 202,
12828        };
12829        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12830        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
12831            state.pid = Some(successor.pid);
12832            state.process_start_time = Some(successor.start_time);
12833        })
12834        .unwrap();
12835        assert!(!module.record_deliberate_severance(severed).unwrap());
12836
12837        let exit_report = apply_deliberate_severance_marker(
12838            &module.inner.snapshot,
12839            Some(successor),
12840            ExitReport {
12841                kind: ExitKind::Crash,
12842                code: Some(1),
12843                signal: None,
12844                at_ms: 1,
12845            },
12846        );
12847
12848        assert_eq!(exit_report.kind, ExitKind::Crash);
12849    }
12850
12851    #[tokio::test]
12852    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
12853        let registry = Registry::default();
12854        let supervisor = Supervisor::new_for_test(
12855            Arc::new(Registry::default()),
12856            RestartPolicy::new(3, Duration::ZERO),
12857        );
12858        let runtime = supervisor.runtime_config();
12859        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12860        let spec = ModuleSpec {
12861            module_id: "drain-deliberate-severance".to_string(),
12862            program: fake_aft_stub_path(),
12863            args: Vec::new(),
12864            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12865            reserved: false,
12866            reserved_prefixes: Vec::new(),
12867            protocol: ModuleProtocol::Subc,
12868            overlap: Default::default(),
12869        };
12870        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
12871        let process = ProcessIdentity {
12872            pid: 41,
12873            start_time: 101,
12874        };
12875        child.process_identity = Some(process);
12876        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12877            state.pid = Some(process.pid);
12878            state.process_start_time = Some(process.start_time);
12879        })
12880        .unwrap();
12881        record_deliberate_severance(&snapshot, process).unwrap();
12882
12883        drain_child_to_state(
12884            &spec.module_id,
12885            spec.protocol,
12886            // The child exits on its own; no signal may change the exit this
12887            // test classifies.
12888            StopNotice::SentOverConnection,
12889            &registry,
12890            None,
12891            &snapshot,
12892            &runtime.terminal_ring,
12893            &runtime.spawn_events,
12894            child,
12895            Duration::from_secs(1),
12896            ModuleState::Stopped,
12897            Some(false),
12898        )
12899        .await
12900        .unwrap();
12901
12902        let state = lock_snapshot(&snapshot).unwrap();
12903        assert_eq!(
12904            state.last_exit.as_ref().map(|exit| exit.kind),
12905            Some(ExitKind::DeliberateSeverance)
12906        );
12907        assert_eq!(state.lifetime_restarts, 1);
12908        assert_eq!(state.crash_restarts.len(), 0);
12909        drop(state);
12910        let history = runtime.terminal_ring.lock().unwrap().snapshot();
12911        assert_eq!(
12912            history.entries[0].exit_kind,
12913            subc_control::TerminalExitKind::DeliberateSeverance
12914        );
12915    }
12916
12917    #[tokio::test]
12918    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
12919        let registry = Registry::default();
12920        let supervisor = Supervisor::new_for_test(
12921            Arc::new(Registry::default()),
12922            RestartPolicy::new(3, Duration::ZERO),
12923        );
12924        let runtime = supervisor.runtime_config();
12925        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12926        let spec = ModuleSpec {
12927            module_id: "ordinary-drain".to_string(),
12928            program: fake_aft_stub_path(),
12929            args: Vec::new(),
12930            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12931            reserved: false,
12932            reserved_prefixes: Vec::new(),
12933            protocol: ModuleProtocol::Subc,
12934            overlap: Default::default(),
12935        };
12936        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
12937
12938        drain_child_to_state(
12939            &spec.module_id,
12940            spec.protocol,
12941            // The child exits on its own; no signal may change the exit this
12942            // test classifies.
12943            StopNotice::SentOverConnection,
12944            &registry,
12945            None,
12946            &snapshot,
12947            &runtime.terminal_ring,
12948            &runtime.spawn_events,
12949            child,
12950            Duration::from_secs(1),
12951            ModuleState::Stopped,
12952            Some(false),
12953        )
12954        .await
12955        .unwrap();
12956
12957        let state = lock_snapshot(&snapshot).unwrap();
12958        assert_eq!(
12959            state.last_exit.as_ref().map(|exit| exit.kind),
12960            Some(ExitKind::Crash)
12961        );
12962        assert_eq!(state.lifetime_restarts, 0);
12963        assert_eq!(state.crash_restarts.len(), 0);
12964    }
12965
12966    #[test]
12967    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
12968        // The server's generic fatal-routing branch only knows that the
12969        // connection failed; it does not know that the daemon deliberately
12970        // initiated a process-killing severance. Keep this seam explicit so a
12971        // future connection error path cannot silently reintroduce the stale
12972        // exemption that mislabels a later genuine crash.
12973        assert!(!include_str!("server.rs")
12974            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
12975    }
12976
12977    /// The `route.closed` `drained` value must be the quiescence wait's own
12978    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
12979    /// measurement at all and `false` is the one honest constant. This is the exact
12980    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
12981    /// on every return path, including the one that used to return early via `?`
12982    /// with `route.closing` already sent and no `route.closed` ever following.
12983    #[test]
12984    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
12985        assert!(drained_after_quiescence_wait(&Ok(true)));
12986        assert!(!drained_after_quiescence_wait(&Ok(false)));
12987        assert!(!drained_after_quiescence_wait(&Err(
12988            SuperviseError::StatePoisoned { module_id: None }
12989        )));
12990    }
12991
12992    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
12993    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
12994    /// already reaped out-of-band) still leaves a terminal record rather than none
12995    /// at all. Triggering the real `wait()` I/O error from an integration test would
12996    /// need a genuine already-reaped-child race, which is OS-specific and not
12997    /// something this suite attempts elsewhere; this test instead verifies the
12998    /// record produced for that arm end-to-end through the real `TerminalRing`, and
12999    /// the call site itself is verified by inspection to sit in that exact arm.
13000    #[test]
13001    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13002        let ring = Arc::new(Mutex::new(TerminalRing::new(
13003            TerminalRingConfig::default(),
13004            0,
13005        )));
13006        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13007
13008        let snapshot = ring.lock().unwrap().snapshot();
13009        assert_eq!(snapshot.entries.len(), 1);
13010        let entry = &snapshot.entries[0];
13011        assert_eq!(entry.exit_code, None);
13012        assert_eq!(entry.exit_signal, None);
13013        assert_eq!(entry.disposition, TerminalDisposition::Failed);
13014    }
13015
13016    #[test]
13017    fn wait_error_exit_path_preserves_spawn_event_density() {
13018        let feed = super::SpawnEventFeed::default();
13019        feed.configure_incarnation("wait-error-density".to_string());
13020        feed.emit_spawned("wait-error", 41, 1);
13021        let ring = Arc::new(Mutex::new(TerminalRing::new(
13022            TerminalRingConfig::default(),
13023            0,
13024        )));
13025
13026        record_wait_error_terminal("wait-error", &ring, &feed);
13027        feed.emit_spawned("after-wait-error", 42, 2);
13028
13029        let state = feed.0.lock().unwrap();
13030        let sequences = state
13031            .events
13032            .iter()
13033            .map(|event| event.cursor.seq)
13034            .collect::<Vec<_>>();
13035        assert_eq!(sequences, vec![1, 2, 3]);
13036        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13037        assert_eq!(state.events[1].exit_code, None);
13038        assert_eq!(state.events[1].exit_signal, None);
13039    }
13040
13041    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
13042    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
13043    /// not a clean exit it never actually observed.
13044    #[test]
13045    fn wait_error_exit_report_is_classified_as_a_crash() {
13046        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13047    }
13048}
13049
13050#[cfg(all(test, unix))]
13051mod health_event_tests {
13052    use super::*;
13053    use cortexkit_test_support::ScratchDir;
13054    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13055
13056    struct Actor {
13057        module: SupervisedModule,
13058        registry: Arc<Registry>,
13059        forwarding: Arc<ForwardingTable>,
13060        spec: ModuleSpec,
13061        health: HealthConfig,
13062        rx: mpsc::Receiver<crate::router::OutboundFrame>,
13063        _home: ScratchDir,
13064    }
13065
13066    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13067        // OS exec and socket readiness are real I/O. Stay runnable while waiting
13068        // for an observed condition, rather than letting paused time auto-advance
13069        // deadlines or assuming a fixed number of yields finishes the work.
13070        let deadline = std::time::Instant::now() + Duration::from_secs(10);
13071        let tick = Instant::now();
13072        loop {
13073            if let Some(value) = observe() {
13074                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
13075                return value;
13076            }
13077            assert!(
13078                std::time::Instant::now() < deadline,
13079                "timed out waiting for {reason}"
13080            );
13081            tokio::task::yield_now().await;
13082        }
13083    }
13084
13085    impl Actor {
13086        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
13087            let home = ScratchDir::new("health-events");
13088            let registry = Arc::new(Registry::default());
13089            let forwarding = Arc::new(ForwardingTable::default());
13090            let health = HealthConfig {
13091                cadence,
13092                deadline: Duration::from_secs(3600),
13093                ..HealthConfig::default()
13094            };
13095            let supervisor =
13096                Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
13097                    .with_forwarding(forwarding.clone())
13098                    .with_health_config(health.clone());
13099            let spec = ModuleSpec {
13100                module_id: "event-health-child".into(),
13101                program: PathBuf::from("/bin/sh"),
13102                // The process stays alive but has no real bus peer. Registrations
13103                // below exercise the same registry writes as accepted HELLOs.
13104                args: vec!["-c".into(), "exec sleep 600".into()],
13105                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13106                    .into_iter()
13107                    .map(|key| (key.into(), home.path().display().to_string()))
13108                    .collect(),
13109                reserved: false,
13110                reserved_prefixes: vec![],
13111                protocol,
13112                overlap: Default::default(),
13113            };
13114            let module = supervisor.spawn(spec.clone()).unwrap();
13115            let (_, rx) = mpsc::channel(8);
13116            let actor = Self {
13117                module,
13118                registry,
13119                forwarding,
13120                spec,
13121                health,
13122                rx,
13123                _home: home,
13124            };
13125            actor
13126                .wait_for_select(|checkpoint| checkpoint.generation > 0)
13127                .await;
13128            actor
13129        }
13130
13131        fn turns(&self) -> u64 {
13132            lock_snapshot(&self.module.inner.snapshot)
13133                .unwrap()
13134                .actor_turns
13135        }
13136
13137        async fn wait_for_select(
13138            &self,
13139            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13140        ) -> ActorSelectCheckpoint {
13141            wait_for("actor select checkpoint", || {
13142                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13143                state.actor_select.filter(|checkpoint| {
13144                    checkpoint.generation == state.spawn_generation
13145                        && state.state == ModuleState::Running
13146                        && state.process_alive
13147                        && state.reported_pid().is_some()
13148                        && expected(checkpoint)
13149                })
13150            })
13151            .await
13152        }
13153
13154        fn hello(&mut self, connection: u64, health: bool) {
13155            let manifest =
13156                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13157                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13158                    .build();
13159            let connection = ConnectionId::new(connection);
13160            let (tx, rx) = mpsc::channel(8);
13161            self.rx = rx;
13162            self.forwarding
13163                .register_module_connection(
13164                    connection,
13165                    self.spec.module_id.clone(),
13166                    subc_protocol::PROTOCOL_VERSION,
13167                    Concurrency::ModuleManaged,
13168                    FrameSink::new(tx),
13169                )
13170                .unwrap();
13171            self.registry
13172                .register_with_control_ops(
13173                    manifest,
13174                    subc_protocol::PROTOCOL_VERSION,
13175                    connection,
13176                    if health {
13177                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13178                    } else {
13179                        vec![]
13180                    },
13181                )
13182                .unwrap();
13183        }
13184
13185        async fn probe(&mut self) -> crate::router::OutboundFrame {
13186            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13187                Ok(frame) => Some(frame),
13188                Err(mpsc::error::TryRecvError::Empty) => None,
13189                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13190            })
13191            .await;
13192            let body: Value = serde_json::from_slice(&frame.body).unwrap();
13193            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13194            frame
13195        }
13196
13197        fn no_probe(&mut self) {
13198            assert!(matches!(
13199                self.rx.try_recv(),
13200                Err(mpsc::error::TryRecvError::Empty)
13201            ));
13202        }
13203
13204        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13205            self.forwarding
13206                .complete_module_control_rpc(
13207                    ConnectionId::new(connection),
13208                    frame.header.corr,
13209                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13210                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13211                        status: HealthStatus::Ok,
13212                        detail: None,
13213                        metrics: None,
13214                    }),
13215                )
13216                .unwrap();
13217        }
13218    }
13219
13220    #[tokio::test(start_paused = true)]
13221    async fn no_health_children_park_for_sixty_seconds() {
13222        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13223        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13224        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13225        unadvertised.hello(1, false);
13226        // Include any one-off registration wake in the budget; receiving a
13227        // no-health HELLO must not turn a parked child into a polling child.
13228        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
13229        // a single 60 s jump would only observe one expired timer.
13230        for _ in 0..6000 {
13231            tokio::time::advance(Duration::from_millis(10)).await;
13232            tokio::task::yield_now().await;
13233        }
13234        for actor in [&none, &unregistered, &unadvertised] {
13235            assert!(
13236                actor.turns() <= 5,
13237                "no-health actor woke {} times",
13238                actor.turns()
13239            );
13240            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13241        }
13242    }
13243
13244    #[tokio::test(start_paused = true)]
13245    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13246        // With a zero cadence the first probe is due at once, so this measures
13247        // only how fast the registration notification starts health probing,
13248        // not the normal 30-33 s wait before a first probe (checked below).
13249        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13250        let tick = Instant::now();
13251        actor.hello(1, true);
13252        actor.probe().await;
13253        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13254    }
13255
13256    #[tokio::test(start_paused = true)]
13257    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13258        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13259        let hello_tick = Instant::now();
13260        actor.hello(1, true);
13261        let armed = actor
13262            .wait_for_select(|checkpoint| {
13263                checkpoint.registered_connection == Some(ConnectionId::new(1))
13264                    && checkpoint.next_probe_at.is_some()
13265            })
13266            .await;
13267        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13268        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13269        let first_deadline = hello_tick + delay;
13270        assert_eq!(armed.next_probe_at, Some(first_deadline));
13271        tokio::time::advance(Duration::from_secs(29)).await;
13272        let turns = actor.turns();
13273        actor
13274            .registry
13275            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13276            .unwrap();
13277        let updated = actor
13278            .wait_for_select(|checkpoint| checkpoint.turn > turns)
13279            .await;
13280        assert_eq!(
13281            updated.next_probe_at,
13282            Some(first_deadline),
13283            "catalog update must not reset cadence"
13284        );
13285        actor.no_probe();
13286        tokio::time::advance(first_deadline - Instant::now()).await;
13287        let frame = actor.probe().await;
13288        actor.answer(1, frame);
13289        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13290        let next_deadline = Instant::now() + next;
13291        let rearmed = actor
13292            .wait_for_select(|checkpoint| {
13293                checkpoint
13294                    .next_probe_at
13295                    .is_some_and(|deadline| deadline > first_deadline)
13296            })
13297            .await;
13298        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
13299        tokio::time::advance(next - Duration::from_nanos(1)).await;
13300        actor.no_probe();
13301        tokio::time::advance(Duration::from_nanos(1)).await;
13302        actor.probe().await;
13303    }
13304
13305    #[tokio::test(start_paused = true)]
13306    async fn disconnect_disarms_and_reconnect_rearms_health() {
13307        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13308        actor.hello(1, true);
13309        actor
13310            .wait_for_select(|checkpoint| {
13311                checkpoint.registered_connection == Some(ConnectionId::new(1))
13312                    && checkpoint.next_probe_at.is_some()
13313            })
13314            .await;
13315        let turns = actor.turns();
13316        let tick = Instant::now();
13317        actor
13318            .registry
13319            .deregister_connection(ConnectionId::new(1))
13320            .unwrap();
13321        let disarmed = actor
13322            .wait_for_select(|checkpoint| {
13323                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
13324            })
13325            .await;
13326        assert_eq!(disarmed.next_probe_at, None);
13327        assert_eq!(disarmed.wake_after, None);
13328        assert_eq!(
13329            Instant::now(),
13330            tick,
13331            "disconnect must disarm in the notification tick"
13332        );
13333        tokio::time::advance(Duration::from_secs(60)).await;
13334        actor.no_probe();
13335        let reconnect_tick = Instant::now();
13336        actor.hello(2, true);
13337        let rearmed = actor
13338            .wait_for_select(|checkpoint| {
13339                checkpoint.registered_connection == Some(ConnectionId::new(2))
13340                    && checkpoint.next_probe_at.is_some()
13341            })
13342            .await;
13343        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13344        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
13345        tokio::time::advance(delay).await;
13346        actor.probe().await;
13347    }
13348
13349    #[tokio::test(start_paused = true)]
13350    async fn reregistration_without_health_disarms_the_old_schedule() {
13351        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13352        actor.hello(1, true);
13353        actor
13354            .wait_for_select(|checkpoint| {
13355                checkpoint.registered_connection == Some(ConnectionId::new(1))
13356                    && checkpoint.next_probe_at.is_some()
13357            })
13358            .await;
13359        actor
13360            .registry
13361            .deregister_connection(ConnectionId::new(1))
13362            .unwrap();
13363        actor.hello(2, false);
13364        let disarmed = actor
13365            .wait_for_select(|checkpoint| {
13366                checkpoint.registered_connection == Some(ConnectionId::new(2))
13367            })
13368            .await;
13369        assert_eq!(disarmed.next_probe_at, None);
13370        assert_eq!(disarmed.wake_after, None);
13371        tokio::time::advance(Duration::from_secs(60)).await;
13372        actor.no_probe();
13373    }
13374
13375    #[tokio::test(start_paused = true)]
13376    async fn restart_waits_for_its_own_hello_before_rearming_health() {
13377        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13378        actor.hello(1, true);
13379        let armed = actor
13380            .wait_for_select(|checkpoint| {
13381                checkpoint.registered_connection == Some(ConnectionId::new(1))
13382                    && checkpoint.next_probe_at.is_some()
13383            })
13384            .await;
13385        // Simulate the old process's connection closing before the restart
13386        // tears the process down. The replacement process has started but has
13387        // not sent its HELLO, so it is not registered.
13388        actor
13389            .registry
13390            .deregister_connection(ConnectionId::new(1))
13391            .unwrap();
13392        actor.module.restart(Some(0)).await.unwrap();
13393        let parked = actor
13394            .wait_for_select(|checkpoint| {
13395                checkpoint.generation > armed.generation
13396                    && checkpoint.registered_connection.is_none()
13397                    && checkpoint.next_probe_at.is_none()
13398            })
13399            .await;
13400        assert_eq!(parked.wake_after, None);
13401        actor.rx.close();
13402        // Drain the stop notice the restart queued for the old process's connection,
13403        // so it isn't mistaken for traffic to the replacement.
13404        while actor.rx.try_recv().is_ok() {}
13405        let turns = actor.turns();
13406        tokio::time::advance(Duration::from_secs(60)).await;
13407        let after = actor.turns();
13408        eprintln!(
13409            "restart park turns: before={turns} after={after} delta={}",
13410            after - turns
13411        );
13412        assert_eq!(after, turns, "replacement must park before HELLO");
13413        let hello_tick = Instant::now();
13414        actor.hello(2, true);
13415        let rearmed = actor
13416            .wait_for_select(|checkpoint| {
13417                checkpoint.registered_connection == Some(ConnectionId::new(2))
13418                    && checkpoint.next_probe_at.is_some()
13419            })
13420            .await;
13421        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13422        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
13423        tokio::time::advance(delay).await;
13424        actor.probe().await;
13425    }
13426
13427    #[tokio::test(start_paused = true)]
13428    async fn rescan_adds_and_removes_http_health_without_polling() {
13429        use tokio::{
13430            io::{AsyncReadExt, AsyncWriteExt},
13431            net::TcpListener,
13432        };
13433        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13434        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
13435        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
13436        // One millisecond separates successive probes without a polling delay.
13437        actor.health.cadence = Duration::from_millis(1);
13438        let (started_tx, mut started_rx) = mpsc::channel(8);
13439        let peer = tokio::spawn(async move {
13440            loop {
13441                let (mut socket, _) = listener.accept().await.unwrap();
13442                let mut request = [0; 1024];
13443                assert!(socket.read(&mut request).await.unwrap() > 0);
13444                started_tx.send(()).await.unwrap();
13445                socket
13446                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
13447                    .await
13448                    .unwrap();
13449            }
13450        });
13451        let tick = Instant::now();
13452        let turns = actor.turns();
13453        actor
13454            .module
13455            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13456            .await
13457            .unwrap();
13458        let armed = actor
13459            .wait_for_select(|checkpoint| {
13460                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
13461            })
13462            .await;
13463        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
13464        assert_eq!(Instant::now(), tick);
13465        tokio::time::advance(Duration::from_millis(1)).await;
13466        let mut started = false;
13467        wait_for("successful HTTP probe", || {
13468            started |= started_rx.try_recv().is_ok();
13469            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
13470                .then_some(())
13471        })
13472        .await;
13473        assert!(started, "added HTTP check must start at its first deadline");
13474        assert_eq!(
13475            actor.module.status().unwrap().health.status,
13476            SupervisorHealthStatus::Ok
13477        );
13478        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
13479        actor.health.http = None;
13480        let turns = actor.turns();
13481        actor
13482            .module
13483            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13484            .await
13485            .unwrap();
13486        let parked = actor
13487            .wait_for_select(|checkpoint| {
13488                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
13489            })
13490            .await;
13491        assert_eq!(parked.wake_after, None);
13492        let turns = actor.turns();
13493        tokio::time::advance(Duration::from_secs(60)).await;
13494        assert_eq!(
13495            actor.turns(),
13496            turns,
13497            "removed HTTP check must park the actor"
13498        );
13499        assert!(
13500            started_rx.try_recv().is_err(),
13501            "removed HTTP check must stay disarmed"
13502        );
13503        assert_eq!(
13504            actor.module.status().unwrap().health,
13505            ModuleHealthStatus::default()
13506        );
13507        peer.abort();
13508    }
13509}
13510
13511#[cfg(test)]
13512mod drain_busy_reprobe_tests {
13513    //! The drain's busy-gauge wait, driven against an in-process module
13514    //! connection whose health answers the test scripts. Time is paused, so
13515    //! every elapsed duration below is exact virtual time.
13516    use super::*;
13517    use std::sync::Mutex as StdMutex;
13518    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13519
13520    const MODULE_ID: &str = "drain-reprobe-module";
13521    const CONNECTION: u64 = 7;
13522    const GAUGE: &str = "runs_in_flight";
13523    /// The production default for both the health cadence and the drain
13524    /// ceiling, which is the pairing that used to cost a full ceiling.
13525    const CADENCE: Duration = Duration::from_secs(30);
13526    const CEILING: Duration = Duration::from_secs(30);
13527
13528    /// How the scripted module answers each `health.check`.
13529    #[derive(Clone, Copy)]
13530    enum Answer {
13531        /// Report the gauge busy until this instant, quiet from then on, after
13532        /// taking `latency` to answer.
13533        BusyUntil { until: Instant, latency: Duration },
13534        /// Never answer.
13535        Silent,
13536    }
13537
13538    struct Drain {
13539        forwarding: Arc<ForwardingTable>,
13540        runtime: SupervisorRuntimeConfig,
13541        endpoint: crate::ModuleEndpointId,
13542        /// When each probe reached the module.
13543        probes: Arc<StdMutex<Vec<Instant>>>,
13544        responder: tokio::task::JoinHandle<()>,
13545    }
13546
13547    impl Drain {
13548        fn start(answer: Answer) -> Self {
13549            let registry = Arc::new(Registry::default());
13550            let forwarding = Arc::new(ForwardingTable::default());
13551            let health = HealthConfig {
13552                cadence: CADENCE,
13553                ..HealthConfig::default()
13554            };
13555            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
13556                .with_forwarding(forwarding.clone())
13557                .with_health_config(health)
13558                .runtime_config();
13559            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
13560            let endpoint = forwarding
13561                .register_module_connection(
13562                    ConnectionId::new(CONNECTION),
13563                    MODULE_ID.to_string(),
13564                    subc_protocol::PROTOCOL_VERSION,
13565                    Concurrency::ModuleManaged,
13566                    FrameSink::new(tx),
13567                )
13568                .unwrap();
13569            let probes = Arc::new(StdMutex::new(Vec::new()));
13570            let responder = tokio::spawn({
13571                let forwarding = forwarding.clone();
13572                let probes = probes.clone();
13573                async move {
13574                    while let Some(frame) = rx.recv().await {
13575                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
13576                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13577                        probes.lock().unwrap().push(Instant::now());
13578                        let Answer::BusyUntil { until, latency } = answer else {
13579                            continue;
13580                        };
13581                        sleep(latency).await;
13582                        let busy = u64::from(Instant::now() < until);
13583                        // A probe abandoned at its deadline has no RPC left to
13584                        // complete, so a late answer may be refused.
13585                        let _ = forwarding.complete_module_control_rpc(
13586                            ConnectionId::new(CONNECTION),
13587                            frame.header.corr,
13588                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13589                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13590                                status: HealthStatus::Ok,
13591                                detail: None,
13592                                metrics: Some(serde_json::json!({ GAUGE: busy })),
13593                            }),
13594                        );
13595                    }
13596                }
13597            });
13598            Self {
13599                forwarding,
13600                runtime,
13601                endpoint,
13602                probes,
13603                responder,
13604            }
13605        }
13606
13607        /// Run one by-id drain wait with `ceiling`, returning its verdict and
13608        /// how long it took.
13609        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
13610            let started = Instant::now();
13611            let drained = wait_for_forwarding_quiescence(
13612                &self.forwarding,
13613                MODULE_ID,
13614                &self.runtime,
13615                self.endpoint,
13616                started + ceiling,
13617                &[GAUGE.to_string()],
13618                DrainScope::Active,
13619            )
13620            .await
13621            .unwrap();
13622            (drained, started.elapsed())
13623        }
13624
13625        fn probes(&self) -> Vec<Instant> {
13626            self.probes.lock().unwrap().clone()
13627        }
13628    }
13629
13630    impl Drop for Drain {
13631        fn drop(&mut self) {
13632            self.responder.abort();
13633        }
13634    }
13635
13636    #[tokio::test(start_paused = true)]
13637    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
13638        let drain = Drain::start(Answer::BusyUntil {
13639            until: Instant::now() + Duration::from_secs(1),
13640            latency: Duration::ZERO,
13641        });
13642        let (drained, elapsed) = drain.run(CEILING).await;
13643        assert!(
13644            drained,
13645            "gauges that read zero during the drain must let it finish drained, \
13646             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
13647        );
13648        assert!(
13649            elapsed < Duration::from_secs(3),
13650            "drain took {elapsed:?} after the gauges went quiet at 1s"
13651        );
13652        let probes = drain.probes();
13653        assert!(
13654            probes.len() >= 2,
13655            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
13656        );
13657    }
13658
13659    #[tokio::test(start_paused = true)]
13660    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
13661        let started = Instant::now();
13662        let drain = Drain::start(Answer::BusyUntil {
13663            until: started + Duration::from_secs(3600),
13664            latency: Duration::ZERO,
13665        });
13666        let (drained, elapsed) = drain.run(CEILING).await;
13667        assert!(!drained, "a module busy throughout must not drain");
13668        assert!(
13669            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
13670            "a busy module is held until the ceiling and no longer: {elapsed:?}"
13671        );
13672        // Re-probing is paced by the interval, not a busy loop: at most one
13673        // probe per interval across the ceiling, plus the first at time zero.
13674        let probes = drain.probes();
13675        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
13676        assert!(
13677            probes.len() as u128 <= max_probes,
13678            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
13679            probes.len()
13680        );
13681        for pair in probes.windows(2) {
13682            assert!(
13683                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
13684                "consecutive probes {:?} apart",
13685                pair[1] - pair[0]
13686            );
13687        }
13688    }
13689
13690    #[tokio::test(start_paused = true)]
13691    async fn a_slow_probe_is_never_overlapped_by_the_next() {
13692        // Each answer takes longer than the re-probe interval; the next probe
13693        // must wait for the previous answer and then the interval.
13694        let latency = Duration::from_secs(1);
13695        let drain = Drain::start(Answer::BusyUntil {
13696            until: Instant::now() + Duration::from_secs(3600),
13697            latency,
13698        });
13699        let (drained, _) = drain.run(Duration::from_secs(5)).await;
13700        assert!(!drained);
13701        let probes = drain.probes();
13702        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
13703        for pair in probes.windows(2) {
13704            assert!(
13705                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
13706                "a probe started {:?} after the previous one, before it could finish",
13707                pair[1] - pair[0]
13708            );
13709        }
13710    }
13711
13712    #[tokio::test(start_paused = true)]
13713    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
13714        let drain = Drain::start(Answer::Silent);
13715        let ceiling = Duration::from_secs(2);
13716        let (drained, elapsed) = drain.run(ceiling).await;
13717        assert!(!drained, "an unanswered probe counts as busy");
13718        assert!(
13719            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
13720            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
13721        );
13722        assert_eq!(
13723            drain.probes().len(),
13724            1,
13725            "the unanswered probe holds the only slot until the ceiling"
13726        );
13727    }
13728}
13729
13730#[cfg(test)]
13731mod health_evidence_tests {
13732    use super::{HealthProbeError, HealthProbeEvidence};
13733    use std::collections::HashSet;
13734
13735    /// The evidential asymmetry, asserted rather than described.
13736    ///
13737    /// Exactly ONE observation is proof a module cannot serve, and the one that
13738    /// fires under CPU starvation is not it. Before the split, all fifteen
13739    /// construction sites collapsed into a single String, so a timeout carried the
13740    /// same weight as a dead lane -- which is how a healthy module was restarted
13741    /// three times in one day.
13742    #[test]
13743    fn only_a_dead_lane_is_proof_of_death() {
13744        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
13745        // Three non-proof classes, each for a different reason: silence is
13746        // consistent with health, a bad answer proves the module ALIVE, and a
13747        // daemon-side fault never reached the module at all.
13748        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
13749        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
13750        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
13751    }
13752
13753    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
13754    ///
13755    /// A shared label renders two different observations identically in the line an
13756    /// operator reads after an unexplained restart -- the exact confusion this
13757    /// change removes.
13758    #[test]
13759    fn every_evidence_class_has_a_distinct_label() {
13760        let labels = [
13761            HealthProbeError::lane_dead("").label(),
13762            HealthProbeError::no_answer("").label(),
13763            HealthProbeError::bad_answer("").label(),
13764            HealthProbeError::misconfigured("").label(),
13765        ];
13766        let unique: HashSet<_> = labels.iter().collect();
13767        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
13768    }
13769
13770    /// The class is additional information, not a replacement.
13771    ///
13772    /// An operator needs both "this was silence" and the specific text saying how
13773    /// long we waited; a classification that swallowed the message would trade one
13774    /// missing distinction for another.
13775    #[test]
13776    fn classification_preserves_the_original_message() {
13777        let err = HealthProbeError::no_answer("module did not answer within 5s");
13778        assert_eq!(err.to_string(), "module did not answer within 5s");
13779        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13780    }
13781}
13782
13783#[cfg(test)]
13784mod health_tombstone_tests {
13785    use std::{path::PathBuf, sync::Arc, time::Duration};
13786
13787    use subc_protocol::{
13788        manifest::Concurrency,
13789        session::{HealthStatus, ModuleControlResponse},
13790    };
13791    use tokio::sync::mpsc;
13792
13793    use super::{
13794        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
13795        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
13796    };
13797    use crate::{
13798        control::ControlHandler,
13799        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
13800        registry::{ConnectionId, Registry},
13801        router::FrameSink,
13802    };
13803
13804    struct ProbeHarness {
13805        spec: ModuleSpec,
13806        runtime: SupervisorRuntimeConfig,
13807        forwarding: Arc<ForwardingTable>,
13808        module_connection: ConnectionId,
13809        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13810        handler: ControlHandler,
13811        module: super::SupervisedModule,
13812    }
13813
13814    fn probe_harness() -> ProbeHarness {
13815        let registry = Arc::new(Registry::default());
13816        let forwarding = Arc::new(ForwardingTable::default());
13817        let supervisor_handle = super::SupervisorHandle::new();
13818        let health = HealthConfig {
13819            http: None,
13820            cadence: Duration::from_secs(30),
13821            deadline: Duration::from_secs(5),
13822            failure_threshold: 3,
13823            on_degraded: HealthAction::Report,
13824            on_failing: HealthAction::Report,
13825            critical: false,
13826        };
13827        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
13828            .with_forwarding(Arc::clone(&forwarding))
13829            .with_handle(supervisor_handle.clone())
13830            .with_health_config(health);
13831        let spec = ModuleSpec {
13832            module_id: "late-health-module".to_string(),
13833            program: PathBuf::from("disabled-module"),
13834            args: Vec::new(),
13835            env: Vec::new(),
13836            reserved: false,
13837            reserved_prefixes: Vec::new(),
13838            protocol: ModuleProtocol::Subc,
13839            overlap: Default::default(),
13840        };
13841        let module = supervisor
13842            .supervise_configured(spec.clone(), false)
13843            .unwrap();
13844        let runtime = supervisor.runtime_config();
13845        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
13846            .with_supervisor(supervisor_handle);
13847        let module_connection = ConnectionId::new(700);
13848        let (module_tx, module_rx) = mpsc::channel(8);
13849        forwarding
13850            .register_module_connection(
13851                module_connection,
13852                spec.module_id.clone(),
13853                subc_protocol::PROTOCOL_VERSION,
13854                Concurrency::ModuleManaged,
13855                FrameSink::new(module_tx),
13856            )
13857            .unwrap();
13858
13859        ProbeHarness {
13860            spec,
13861            runtime,
13862            forwarding,
13863            module_connection,
13864            module_rx,
13865            handler,
13866            module,
13867        }
13868    }
13869
13870    async fn finish_after(
13871        harness: &mut ProbeHarness,
13872        stall: Duration,
13873    ) -> ModuleControlRpcCompletion {
13874        assert!(stall > harness.runtime.health.deadline);
13875        let deadline = harness.runtime.health.deadline;
13876        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
13877        let answer = async {
13878            let frame = harness.module_rx.recv().await.expect("health.check frame");
13879            tokio::time::advance(deadline).await;
13880            tokio::task::yield_now().await;
13881            tokio::time::advance(stall - deadline).await;
13882            harness
13883                .forwarding
13884                .complete_module_control_rpc(
13885                    harness.module_connection,
13886                    frame.header.corr,
13887                    Some("health.check"),
13888                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13889                        status: HealthStatus::Ok,
13890                        detail: None,
13891                        metrics: None,
13892                    }),
13893                )
13894                .unwrap()
13895        };
13896        let (probe_result, completion) = tokio::join!(probe, answer);
13897        let err = probe_result.expect_err("probe must miss its deadline");
13898        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13899        completion
13900    }
13901
13902    async fn time_out_without_answer(harness: &mut ProbeHarness) {
13903        let deadline = harness.runtime.health.deadline;
13904        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
13905        let exhaust_deadline = async {
13906            let _frame = harness.module_rx.recv().await.expect("health.check frame");
13907            tokio::time::advance(deadline).await;
13908            tokio::task::yield_now().await;
13909        };
13910        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
13911        let err = probe_result.expect_err("probe must miss its deadline");
13912        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13913    }
13914
13915    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
13916        let registry = Arc::clone(&harness.module.inner.registry);
13917        let snapshot = Arc::clone(&harness.module.inner.snapshot);
13918        let process_liveness = super::SupervisorProcessLiveness::default();
13919        let mut child = None;
13920        let cycle = super::run_health_probe_cycle(
13921            &harness.spec,
13922            &harness.runtime,
13923            &registry,
13924            &process_liveness,
13925            &snapshot,
13926            &mut child,
13927        );
13928        let peer = async {
13929            let frame = harness.module_rx.recv().await.expect("health.check frame");
13930            if answer {
13931                harness
13932                    .forwarding
13933                    .complete_module_control_rpc(
13934                        harness.module_connection,
13935                        frame.header.corr,
13936                        Some("health.check"),
13937                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13938                            status: HealthStatus::Ok,
13939                            detail: None,
13940                            metrics: Some(serde_json::json!({"ready": true})),
13941                        }),
13942                    )
13943                    .unwrap();
13944            } else {
13945                tokio::time::advance(harness.runtime.health.deadline).await;
13946                tokio::task::yield_now().await;
13947            }
13948        };
13949        tokio::join!(cycle, peer);
13950    }
13951
13952    #[tokio::test(start_paused = true)]
13953    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
13954        let mut harness = probe_harness();
13955        // Drive the probe cycle directly with an in-memory wire peer. Stop the
13956        // disabled module's monitor so only this test owns lifecycle transitions;
13957        // no OS process is launched, and a restart is observed at scheduling.
13958        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
13959        monitor.abort();
13960        let _ = monitor.await;
13961        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
13962            state.enabled = true;
13963            state.state = super::ModuleState::Running;
13964            state.process_alive = true;
13965        })
13966        .unwrap();
13967
13968        run_probe_cycle(&mut harness, true).await;
13969        assert_eq!(
13970            harness.module.status().unwrap().health.status,
13971            super::SupervisorHealthStatus::Ok
13972        );
13973
13974        for failures in 1..harness.runtime.health.failure_threshold {
13975            run_probe_cycle(&mut harness, false).await;
13976            let status = harness.module.status().unwrap();
13977            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
13978            assert_eq!(status.health.consecutive_failures, failures);
13979            assert!(status.health.last_probe_ms.is_some());
13980            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
13981            assert!(status.health.metrics.is_none());
13982            assert_eq!(status.state, super::ModuleState::Running);
13983            assert!(status.process_alive);
13984            assert_eq!(status.restart_count, 0);
13985            assert_eq!(status.lifetime_restarts, 0);
13986            assert!(status.health.last_action.is_none());
13987            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
13988        }
13989
13990        run_probe_cycle(&mut harness, true).await;
13991        let recovered = harness.module.status().unwrap();
13992        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
13993        assert_eq!(recovered.health.consecutive_failures, 0);
13994        assert!(recovered.health.detail.is_none());
13995        assert_eq!(
13996            recovered.health.metrics,
13997            Some(serde_json::json!({"ready": true}))
13998        );
13999        assert_eq!(recovered.lifetime_restarts, 0);
14000
14001        for failures in 1..=harness.runtime.health.failure_threshold {
14002            run_probe_cycle(&mut harness, false).await;
14003            let status = harness.module.status().unwrap();
14004            assert_eq!(status.health.consecutive_failures, failures);
14005            if failures < harness.runtime.health.failure_threshold {
14006                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14007                assert_eq!(status.state, super::ModuleState::Running);
14008                assert_eq!(status.lifetime_restarts, 0);
14009                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14010            } else {
14011                assert_eq!(
14012                    status.health.status,
14013                    super::SupervisorHealthStatus::Unresponsive
14014                );
14015                assert_eq!(status.state, super::ModuleState::Restarting);
14016                assert_eq!(status.restart_count, 1);
14017                assert_eq!(status.lifetime_restarts, 1);
14018                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
14019                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
14020            }
14021        }
14022    }
14023
14024    #[tokio::test(start_paused = true)]
14025    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
14026        let mut harness = probe_harness();
14027
14028        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
14029        let first_latency = match &first {
14030            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14031            other => panic!("late answer was not retained: {other:?}"),
14032        };
14033        assert!(harness.handler.observe_module_control_completion(first));
14034
14035        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
14036        let second_latency = match &second {
14037            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14038            other => panic!("late answer was not retained: {other:?}"),
14039        };
14040        assert!(harness.handler.observe_module_control_completion(second));
14041
14042        assert_eq!(first_latency, Duration::from_secs(8));
14043        assert_eq!(
14044            second_latency - first_latency,
14045            Duration::from_secs(3),
14046            "latency must grow linearly with the additional stall"
14047        );
14048        let health = harness.module.status().unwrap().health;
14049        assert_eq!(health.late_answer_count, 2);
14050        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
14051    }
14052
14053    /// A module that answers every probe late must never march to the kill
14054    /// threshold: the late answer proves it is alive, so it must clear the miss
14055    /// streak the timeout recorded. Without the reset, a CPU-starved module
14056    /// that serves every probe seconds past the deadline accumulates
14057    /// `consecutive_failures` to the threshold and is killed — the exact
14058    /// sequence from the 2026-08-14 aft disable, where the daemon logged
14059    /// "proves the module is alive" five times while counting five misses.
14060    #[tokio::test(start_paused = true)]
14061    async fn late_answer_clears_the_consecutive_failure_streak() {
14062        let mut harness = probe_harness();
14063
14064        // Timeout recorded first: the probe path saw no answer in time.
14065        time_out_without_answer(&mut harness).await;
14066        harness
14067            .module
14068            .record_health_probe_failure_for_test("[no-answer] test miss")
14069            .unwrap();
14070        assert_eq!(
14071            harness.module.status().unwrap().health.consecutive_failures,
14072            1,
14073            "precondition: the miss must be on the streak before the late answer"
14074        );
14075
14076        // The stalled reply then lands: proof of life.
14077        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
14078        assert!(matches!(
14079            late,
14080            ModuleControlRpcCompletion::LateHealthAnswer { .. }
14081        ));
14082        assert!(harness.handler.observe_module_control_completion(late));
14083
14084        let health = harness.module.status().unwrap().health;
14085        assert_eq!(
14086            health.consecutive_failures, 0,
14087            "a late answer is an answer: the streak must reset"
14088        );
14089        assert_eq!(health.late_answer_count, 1);
14090    }
14091
14092    #[tokio::test(start_paused = true)]
14093    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
14094        let mut harness = probe_harness();
14095
14096        for _ in 0..20 {
14097            time_out_without_answer(&mut harness).await;
14098            assert_eq!(
14099                harness.forwarding.health_probe_tombstone_count().unwrap(),
14100                1
14101            );
14102        }
14103    }
14104}
14105
14106#[cfg(test)]
14107mod child_env_tests {
14108    #[cfg(windows)]
14109    use super::{
14110        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
14111    };
14112    #[cfg(windows)]
14113    use std::sync::{Arc, OnceLock};
14114    #[cfg(windows)]
14115    #[test]
14116    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
14117        let spec = spec(Vec::new());
14118        let mut command = Command::new("unused");
14119        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
14120            .unwrap()
14121            .unwrap();
14122        let name = command
14123            .as_std()
14124            .get_envs()
14125            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
14126            .and_then(|(_, value)| value)
14127            .unwrap();
14128        assert_eq!(name, OsStr::new(handoff.name()));
14129        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
14130        assert!(command.as_std().get_envs().any(|(key, value)| key
14131            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14132            && value == Some(OsStr::new("env"))));
14133        assert!(command
14134            .as_std()
14135            .get_envs()
14136            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
14137    }
14138
14139    #[cfg(windows)]
14140    #[test]
14141    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
14142        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
14143        snapshot.process_alive = true;
14144        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
14145        assert_eq!(
14146            observed_launch_nonce_source(&snapshot).as_deref(),
14147            Some("env")
14148        );
14149        let consumed = Arc::new(OnceLock::new());
14150        snapshot.pipe_consumed = Some(consumed.clone());
14151        assert_eq!(
14152            observed_launch_nonce_source(&snapshot).as_deref(),
14153            Some("env")
14154        );
14155        consumed.set(()).unwrap();
14156        assert_eq!(
14157            observed_launch_nonce_source(&snapshot).as_deref(),
14158            Some("pipe")
14159        );
14160        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
14161        assert_eq!(
14162            observed_launch_nonce_source(&snapshot).as_deref(),
14163            Some("env")
14164        );
14165        clear_current_process_facts(&mut snapshot);
14166        assert_eq!(observed_launch_nonce_source(&snapshot), None);
14167    }
14168    use super::{
14169        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
14170        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
14171        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
14172    };
14173    use std::{ffi::OsStr, path::PathBuf};
14174    use tokio::process::Command;
14175
14176    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
14177        ModuleSpec {
14178            module_id: "env-plan".to_string(),
14179            program: PathBuf::from("/nonexistent"),
14180            args: Vec::new(),
14181            env,
14182            reserved: false,
14183            reserved_prefixes: Vec::new(),
14184            protocol: ModuleProtocol::Subc,
14185            overlap: Default::default(),
14186        }
14187    }
14188
14189    #[test]
14190    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
14191        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
14192        let digest = LaunchSpecDigest::from_spec(&base);
14193        let mut reordered = base.clone();
14194        reordered.env.reverse();
14195        reordered.module_id = "different-id".into();
14196        reordered.reserved = true;
14197        reordered.overlap = Default::default();
14198        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
14199        let mut delimiter_collision = base.clone();
14200        delimiter_collision.env[0] = ("A=B".into(), "C".into());
14201        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
14202        let mut changed = base.clone();
14203        changed.protocol = ModuleProtocol::None;
14204        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14205        changed = base.clone();
14206        changed.args.push("--flag".into());
14207        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14208        changed = base.clone();
14209        changed.program = PathBuf::from("/other");
14210        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14211        changed = base.clone();
14212        changed.env[1].1 = "other".into();
14213        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14214        assert!(!format!("{digest:?}").contains("B=C"));
14215        assert!(!format!("{digest:?}").contains("[u8"));
14216    }
14217
14218    #[cfg(unix)]
14219    #[test]
14220    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
14221        use std::os::unix::ffi::OsStringExt;
14222        let mut left = spec(Vec::new());
14223        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
14224        let mut right = left.clone();
14225        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
14226        assert_eq!(
14227            left.program.to_string_lossy(),
14228            right.program.to_string_lossy()
14229        );
14230        assert_ne!(
14231            LaunchSpecDigest::from_spec(&left),
14232            LaunchSpecDigest::from_spec(&right)
14233        );
14234    }
14235
14236    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
14237    /// one still gets its own.
14238    ///
14239    /// This is the narrow goal `env_clear()` was reached for, and the reason the
14240    /// fix is `env_remove` rather than deleting the line: an operator's ambient
14241    /// filter silently becoming an unconfigured module's log level is a real
14242    /// defect, just a much smaller one than clearing the environment.
14243    ///
14244    /// Asserted on the command plan rather than a spawned child because proving
14245    /// the ABSENCE of an inherited variable needs the parent's environment
14246    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
14247    /// removal as `(key, None)`, which is exactly the distinction wanted: not
14248    /// "absent because nobody set it" but "explicitly unset for the child".
14249    #[test]
14250    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
14251        let mut command = Command::new("/nonexistent");
14252        apply_child_env(&mut command, &spec(Vec::new()));
14253        let removed = command
14254            .as_std()
14255            .get_envs()
14256            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
14257        assert!(
14258            removed,
14259            "ambient CK_LOG must be explicitly removed for an unconfigured module"
14260        );
14261
14262        let mut configured = Command::new("/nonexistent");
14263        apply_child_env(
14264            &mut configured,
14265            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
14266        );
14267        let effective = configured
14268            .as_std()
14269            .get_envs()
14270            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
14271            .last()
14272            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14273        assert_eq!(
14274            effective,
14275            Some(Some("debug".to_string())),
14276            "a module's configured CK_LOG must survive the ambient removal"
14277        );
14278    }
14279
14280    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
14281    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
14282    /// the same reason as the CK_LOG test above.
14283    ///
14284    /// The argument is the load-bearing half: a stock binary exits on an
14285    /// unknown flag before it listens, so with `--subc` appended the mode
14286    /// could not supervise the one process it exists for. Found by the first
14287    /// conformance run (nats-server: `flag provided but not defined: -subc`).
14288    #[test]
14289    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
14290        let connection_file = std::path::Path::new("/run/subc-connection.json");
14291        let handle = SupervisorHandle::new();
14292
14293        let mut none_spec = spec(Vec::new());
14294        none_spec.protocol = ModuleProtocol::None;
14295        let mut none = Command::new("/nonexistent");
14296        let none_handoff =
14297            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
14298                .expect("protocol-none spawn args apply");
14299        assert!(
14300            none_handoff.is_none(),
14301            "protocol:none spawn must not receive a nonce descriptor"
14302        );
14303        #[cfg(windows)]
14304        assert!(!none.as_std().get_envs().any(|(key, value)| key
14305            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14306            && value.is_some()));
14307        assert!(
14308            !none.as_std().get_envs().any(|(key, value)| key
14309                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
14310                && value.is_some()),
14311            "protocol:none spawn must not name a nonce descriptor"
14312        );
14313        let none_args: Vec<String> = none
14314            .as_std()
14315            .get_args()
14316            .map(|a| a.to_string_lossy().into_owned())
14317            .collect();
14318        assert!(
14319            !none_args.iter().any(|a| a == SUBC_ARG),
14320            "protocol:none argv must not carry --subc; got {none_args:?}"
14321        );
14322        let none_has_nonce = none
14323            .as_std()
14324            .get_envs()
14325            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
14326        assert!(
14327            !none_has_nonce,
14328            "protocol:none spawn must not receive a launch nonce"
14329        );
14330        let none_has_module_id = none
14331            .as_std()
14332            .get_envs()
14333            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
14334        assert!(
14335            none_has_module_id,
14336            "SUBC_MODULE_ID is inert and stays on every path"
14337        );
14338        assert!(
14339            handle.spawn_nonce(&none_spec.module_id).is_none(),
14340            "no nonce record for a process that will never present one"
14341        );
14342
14343        // Control: the subc-wire path is unchanged by the branch above.
14344        let wire_spec = spec(Vec::new());
14345        let mut wire = Command::new("/nonexistent");
14346        let wire_handoff =
14347            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
14348                .expect("subc-wire spawn args apply");
14349        let wire_fd_env = wire
14350            .as_std()
14351            .get_envs()
14352            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
14353            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14354        #[cfg(unix)]
14355        assert_eq!(
14356            wire_fd_env,
14357            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
14358            "a subc-wire spawn names the pipe it will receive at descriptor 3"
14359        );
14360        #[cfg(windows)]
14361        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
14362        #[cfg(not(any(unix, windows)))]
14363        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
14364        let wire_args: Vec<String> = wire
14365            .as_std()
14366            .get_args()
14367            .map(|a| a.to_string_lossy().into_owned())
14368            .collect();
14369        assert_eq!(
14370            wire_args,
14371            vec![
14372                SUBC_ARG.to_string(),
14373                connection_file.to_string_lossy().into_owned()
14374            ],
14375            "a subc-wire spawn still carries --subc <path>"
14376        );
14377        assert_eq!(
14378            wire.as_std()
14379                .get_envs()
14380                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
14381            !cfg!(unix),
14382            "only Windows supplies the environment nonce"
14383        );
14384        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
14385    }
14386
14387    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
14388    /// spec tries to set it; only a swap candidate carries it.
14389    ///
14390    /// "Set it only on candidates" is not enough, because spawn applies the
14391    /// spec's env verbatim and the daemon's own environment is inherited: either
14392    /// could hand a plain restart the swap role, and a module reading it would
14393    /// warm on its long swap budget while callers wait. Asserted as an explicit
14394    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
14395    /// test above gives.
14396    #[test]
14397    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
14398        let role = |command: &Command| {
14399            command
14400                .as_std()
14401                .get_envs()
14402                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
14403                .last()
14404                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
14405        };
14406        let forged = spec(vec![(
14407            SUBC_SPAWN_ROLE_ENV.to_string(),
14408            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
14409        )]);
14410
14411        let mut plain = Command::new("/nonexistent");
14412        apply_child_env(&mut plain, &forged);
14413        apply_spawn_role(&mut plain, SpawnRole::Plain);
14414        assert_eq!(
14415            role(&plain),
14416            Some(None),
14417            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
14418        );
14419
14420        let mut candidate = Command::new("/nonexistent");
14421        apply_child_env(&mut candidate, &spec(Vec::new()));
14422        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
14423        assert_eq!(
14424            role(&candidate),
14425            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
14426        );
14427    }
14428
14429    /// Daemon-private capture retention keys never reach the child.
14430    ///
14431    /// cortexkit-log exposes retention as a Rust struct with no environment
14432    /// names, so these entries are supervisor metadata. Passing them through
14433    /// would invent a public child-process contract by accident.
14434    #[test]
14435    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
14436        let mut command = Command::new("/nonexistent");
14437        apply_child_env(
14438            &mut command,
14439            &spec(vec![
14440                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
14441                ("KEPT".to_string(), "yes".to_string()),
14442            ]),
14443        );
14444        let keys: Vec<String> = command
14445            .as_std()
14446            .get_envs()
14447            .filter(|(_, value)| value.is_some())
14448            .map(|(key, _)| key.to_string_lossy().into_owned())
14449            .collect();
14450        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
14451        assert!(
14452            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
14453            "daemon-private capture key leaked to the child: {keys:?}"
14454        );
14455    }
14456}
14457
14458#[cfg(test)]
14459mod jitter_tests {
14460    use super::jittered_health_delay;
14461    use std::{collections::HashSet, time::Duration};
14462
14463    /// Module ids drawn from a real fleet, so the dispersal claim is about names
14464    /// that actually occur rather than invented ones.
14465    ///
14466    /// This is a SAMPLE, not a registry: the property under test is that distinct
14467    /// ids disperse, which holds for any set of distinct strings. Several entries
14468    /// are already historical (modules get renamed), and that costs nothing here --
14469    /// but it means a reader must not mistake this for the live module set, and a
14470    /// rename sweep will match it without there being anything to change.
14471    const FLEET: [&str; 14] = [
14472        "aft",
14473        "alfonso-core",
14474        "magic-context",
14475        "broca",
14476        "thalamus",
14477        "quota",
14478        "engram",
14479        "plexus",
14480        "cerebellum",
14481        "astrocyte",
14482        "synapse",
14483        "subc-mcp",
14484        "cortexkit-credentials",
14485        "subc-federation",
14486    ];
14487
14488    /// Probes must not converge after a fleet-wide restart.
14489    ///
14490    /// This is the property the jitter exists for: every module reconnects at
14491    /// once, and without dispersal all fourteen would then probe on the same
14492    /// tick forever. Nothing failed visibly when this went untested -- a
14493    /// convergent fleet still probes correctly, just in a burst, so the symptom
14494    /// is a periodic load spike that looks like whatever else is running.
14495    #[test]
14496    fn probe_delays_disperse_across_the_fleet() {
14497        let cadence = Duration::from_secs(30);
14498        let delays: HashSet<Duration> = FLEET
14499            .iter()
14500            .map(|id| jittered_health_delay(id, 0, cadence))
14501            .collect();
14502        assert_eq!(
14503            delays.len(),
14504            FLEET.len(),
14505            "every supervised module must land on its own probe offset"
14506        );
14507    }
14508
14509    /// The offset may only ever DELAY a probe, never bring it forward.
14510    ///
14511    /// A delay below the cadence would probe a module more often than
14512    /// configured, which is the opposite of what an operator asked for and
14513    /// would tighten the failure budget without anyone changing it.
14514    #[test]
14515    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
14516        let cadence = Duration::from_secs(30);
14517        let span = cadence / 10;
14518        for id in FLEET {
14519            for probe_index in 0..8 {
14520                let delay = jittered_health_delay(id, probe_index, cadence);
14521                assert!(
14522                    delay >= cadence,
14523                    "{id}#{probe_index}: jitter must not shorten the cadence"
14524                );
14525                assert!(
14526                    delay < cadence + span,
14527                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
14528                );
14529            }
14530        }
14531    }
14532
14533    /// A module keeps its offset across daemon restarts.
14534    ///
14535    /// The delay is derived rather than randomised precisely so a restart does
14536    /// not re-roll every module into a fresh chance of collision. A random
14537    /// source would satisfy the dispersal test above and quietly lose this.
14538    #[test]
14539    fn a_module_offset_is_stable_across_restarts() {
14540        let cadence = Duration::from_secs(30);
14541        for id in FLEET {
14542            assert_eq!(
14543                jittered_health_delay(id, 0, cadence),
14544                jittered_health_delay(id, 0, cadence),
14545                "{id}: the same module and probe index must produce the same offset"
14546            );
14547        }
14548    }
14549
14550    /// A zero cadence disables probing rather than producing a busy loop.
14551    #[test]
14552    fn zero_cadence_yields_zero_delay() {
14553        assert_eq!(
14554            jittered_health_delay("aft", 0, Duration::ZERO),
14555            Duration::ZERO
14556        );
14557    }
14558}
14559
14560#[cfg(all(test, target_os = "linux"))]
14561mod cgroup_placement_tests {
14562    use super::{
14563        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
14564        SuperviseError, SupervisedChild,
14565    };
14566    use crate::stderr_tail::{StderrRing, StderrTailConfig};
14567    use cortexkit_test_support::ScratchDir;
14568    use std::{
14569        fs, io,
14570        path::{Path, PathBuf},
14571        sync::{Arc, Mutex},
14572    };
14573    use tokio::process::Command;
14574
14575    #[tokio::test]
14576    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
14577        use super::*;
14578        let dir = ScratchDir::new("unique-spawn-cgroups");
14579        let root = PathBuf::from(format!(
14580            "/sys/fs/cgroup/subc-unique-test-{}-{}",
14581            std::process::id(),
14582            unix_ms_now()
14583        ));
14584        if let Err(error) = fs::create_dir(&root) {
14585            assert!(
14586                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
14587                "required cgroup test cannot execute: {error}"
14588            );
14589            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
14590            return;
14591        }
14592        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
14593        let group_count = || {
14594            fs::read_dir(root.join("subc-modules"))
14595                .unwrap()
14596                .map(|entry| entry.unwrap().file_type().unwrap())
14597                .filter(|kind| kind.is_dir())
14598                .count()
14599        };
14600        let supervisor =
14601            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
14602                .with_cgroup_placement(Some(placement.clone()));
14603        let runtime = supervisor.runtime_config();
14604        let mut spec = ModuleSpec {
14605            module_id: "unique-spawn".into(),
14606            program: PathBuf::from("/bin/sleep"),
14607            args: vec!["60".into()],
14608            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14609                .into_iter()
14610                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
14611                .collect(),
14612            reserved: false,
14613            reserved_prefixes: vec![],
14614            protocol: ModuleProtocol::None,
14615            overlap: Default::default(),
14616        };
14617        let spawn = |spec: &ModuleSpec| {
14618            spawn_child(
14619                spec,
14620                None,
14621                None,
14622                &runtime.stderr_ring,
14623                None,
14624                &runtime.child_roster,
14625                Some(&placement),
14626            )
14627            .unwrap()
14628        };
14629        let mut live = spawn(&spec);
14630        for _ in 0..3 {
14631            // A new process can enter the old slot while retirement is pending.
14632            let next = spawn(&spec);
14633            assert_ne!(live.module_id, next.module_id);
14634            live.start_kill().unwrap();
14635            live.wait().await.unwrap();
14636            live = next;
14637            assert!(
14638                live.child.try_wait().unwrap().is_none(),
14639                "retiring the old slot must not kill the replacement"
14640            );
14641            assert_eq!(
14642                group_count(),
14643                1,
14644                "only the live spawn's cgroup should remain"
14645            );
14646        }
14647        supervisor.begin_daemon_shutdown();
14648        let reap = tokio::spawn(async move {
14649            live.wait().await.unwrap();
14650        });
14651        supervisor
14652            .end_children_for_daemon_shutdown(false, std::future::pending())
14653            .await;
14654        reap.await.unwrap();
14655        assert_eq!(group_count(), 0);
14656        // A normal exit uses the same tree-cleanup path as a killed spawn.
14657        spec.program = PathBuf::from("/bin/true");
14658        spec.args.clear();
14659        let fresh_roster = ChildRoster::default();
14660        let mut short = spawn_child(
14661            &spec,
14662            None,
14663            None,
14664            &runtime.stderr_ring,
14665            None,
14666            &fresh_roster,
14667            Some(&placement),
14668        )
14669        .unwrap();
14670        short.wait().await.unwrap();
14671        assert_eq!(group_count(), 0);
14672        spec.module_id = "_".repeat(255);
14673        let mut long_id = spawn_child(
14674            &spec,
14675            None,
14676            None,
14677            &runtime.stderr_ring,
14678            None,
14679            &fresh_roster,
14680            Some(&placement),
14681        )
14682        .unwrap();
14683        long_id.wait().await.unwrap();
14684        assert_eq!(
14685            group_count(),
14686            0,
14687            "valid long module IDs must not exceed cgroup NAME_MAX"
14688        );
14689        fs::remove_dir(root.join("subc-modules")).unwrap();
14690        fs::remove_dir(root).unwrap();
14691        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
14692    }
14693
14694    #[test]
14695    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
14696        let path = Path::new("/definitely-missing-subc-cgroup");
14697        let mut command = Command::new("true");
14698        let error = apply_cgroup_placement(
14699            &mut command,
14700            &ModuleSpec {
14701                module_id: "broken-cgroup".to_string(),
14702                program: PathBuf::from("true"),
14703                args: Vec::new(),
14704                env: Vec::new(),
14705                reserved: false,
14706                reserved_prefixes: Vec::new(),
14707                protocol: ModuleProtocol::Subc,
14708                overlap: Default::default(),
14709            },
14710            path,
14711        )
14712        .expect_err("a parent cgroup open failure must reject the supervised spawn");
14713        let reason = error.to_string();
14714
14715        assert!(
14716            matches!(error, SuperviseError::Cgroup { .. }),
14717            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
14718        );
14719        assert!(
14720            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
14721            "parent cgroup open failure must name cgroup.procs: {reason}"
14722        );
14723    }
14724
14725    #[tokio::test]
14726    async fn reaping_a_child_removes_its_empty_module_cgroup() {
14727        let root = ScratchDir::new("supervisor-reap-cgroup");
14728        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
14729        let placement = subc_cgroup::prepare_at(&root)
14730            .expect("prepare scratch cgroup root")
14731            .expect("scratch root has a cgroup.procs marker");
14732        let module_id = "reaped-module";
14733        let module = placement
14734            .module_path(module_id)
14735            .expect("create scratch module cgroup");
14736        let child = Command::new("true")
14737            .env("XDG_DATA_HOME", root.path())
14738            .env("XDG_RUNTIME_DIR", root.path())
14739            .env("XDG_CONFIG_HOME", root.path())
14740            .spawn()
14741            .expect("spawn short-lived child");
14742        let pid = child.id().expect("spawned child has pid");
14743        let mut child = SupervisedChild {
14744            child,
14745            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
14746            protocol: ModuleProtocol::Subc,
14747            module_id: module_id.to_string(),
14748            cgroup_placement: Some(placement),
14749            stdout_pump: None,
14750            stderr_pump: None,
14751            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
14752            spawned_at_ms: 0,
14753            spawned_from: PathBuf::from("true"),
14754            launch_spec_digest: LaunchSpecDigest([0; 32]),
14755            spawned_file_identity: None,
14756            process_start_time: None,
14757            process_identity: None,
14758            pid,
14759            roster_guard: None,
14760            #[cfg(target_os = "macos")]
14761            privacy_exec: None,
14762            spawn_failure: None,
14763        };
14764
14765        child.wait().await.expect("reap short-lived child");
14766
14767        assert!(
14768            !module.exists(),
14769            "reaping the supervised child must remove its empty cgroup"
14770        );
14771    }
14772
14773    #[test]
14774    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
14775        let root = ScratchDir::new("supervisor-non-empty-cgroup");
14776        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
14777        let placement = subc_cgroup::prepare_at(&root)
14778            .expect("prepare scratch cgroup root")
14779            .expect("scratch root has a cgroup.procs marker");
14780        let module = placement
14781            .module_path("surviving-module")
14782            .expect("create scratch module cgroup");
14783        fs::write(module.join("surviving-process"), b"still present")
14784            .expect("make scratch cgroup non-empty");
14785        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
14786
14787        remove_module_cgroup(&placement, "surviving-module");
14788
14789        let logs = crate::router::test_log::captured_logs(&logs);
14790        assert!(
14791            module.exists(),
14792            "failed removal must leave the cgroup intact"
14793        );
14794        assert!(
14795            logs.contains("could not remove module cgroup after process exit; continuing teardown")
14796                && logs.contains("surviving-module"),
14797            "best-effort removal must report the failure without returning it: {logs}"
14798        );
14799    }
14800
14801    #[test]
14802    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
14803        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
14804        let reason = SuperviseError::Spawn {
14805            program: PathBuf::from("/bin/true"),
14806            source: io::Error::from_raw_os_error(13),
14807            cgroup_path: Some(cgroup_path.clone()),
14808        }
14809        .to_string();
14810
14811        assert!(
14812            reason.contains(&cgroup_path.display().to_string()),
14813            "a pre_exec spawn failure must name the cgroup path: {reason}"
14814        );
14815    }
14816}
14817
14818#[cfg(test)]
14819mod spawn_subscriber_lag_tests {
14820    use super::*;
14821
14822    /// A subscriber whose connection stops draining is dropped once its frame
14823    /// channel fills. The client must learn that from a terminal Error frame
14824    /// after the frames already queued for it, not from a stream that simply
14825    /// goes quiet.
14826    #[tokio::test]
14827    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
14828        let feed = SpawnEventFeed::default();
14829        feed.configure_incarnation("lag-incarnation".to_string());
14830        // A one-slot connection queue that nobody reads until the emits are
14831        // done: the forwarder parks on it and the subscriber channel fills.
14832        let (tx, mut rx) = mpsc::channel(1);
14833        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
14834            .expect("subscribe");
14835        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
14836        for index in 0..emitted {
14837            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
14838            // Let the forwarder take what it can so the fill point is the
14839            // subscriber channel, not a scheduling accident.
14840            tokio::task::yield_now().await;
14841        }
14842        assert_eq!(
14843            feed.subscriber_count(),
14844            0,
14845            "the lagged subscriber must be removed"
14846        );
14847
14848        let mut data = Vec::new();
14849        let mut last = None;
14850        loop {
14851            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14852                .await
14853                .expect("the forwarder must finish once the subscriber is dropped");
14854            let Some(outbound) = next else { break };
14855            let frame = outbound.frame;
14856            if frame.header.ty == FrameType::StreamData {
14857                assert!(last.is_none(), "no data may follow the terminal frame");
14858                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
14859                data.push(event.cursor.seq);
14860            } else {
14861                assert!(last.is_none(), "exactly one terminal frame");
14862                last = Some(frame);
14863            }
14864        }
14865        assert!(!data.is_empty(), "queued frames drain before the terminal");
14866        for pair in data.windows(2) {
14867            assert_eq!(
14868                pair[1],
14869                pair[0] + 1,
14870                "queued frames arrive dense and in order"
14871            );
14872        }
14873        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
14874        assert_eq!(terminal.header.ty, FrameType::Error);
14875        assert_eq!(terminal.header.corr, 7);
14876        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
14877        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
14878        let detail = body.detail.expect("lagged error carries detail");
14879        assert_eq!(
14880            detail["first_undelivered_cursor"]["seq"],
14881            data.last().unwrap() + 1,
14882            "the named cursor is the first event the subscriber did not receive"
14883        );
14884        assert_eq!(
14885            detail["first_undelivered_cursor"]["daemon_incarnation"],
14886            "lag-incarnation"
14887        );
14888    }
14889}
14890
14891#[cfg(test)]
14892mod terminal_history_read_concurrency_tests {
14893    use super::*;
14894    use crate::terminal_journal::read_pause;
14895    use cortexkit_test_support::ScratchDir;
14896    use std::sync::mpsc as std_mpsc;
14897
14898    fn journaled_ring(
14899        journal: &Arc<crate::terminal_journal::TerminalJournal>,
14900    ) -> Arc<Mutex<TerminalRing>> {
14901        Arc::new(Mutex::new(
14902            TerminalRing::new(TerminalRingConfig::default(), 1)
14903                .with_journal(Some(Arc::clone(journal))),
14904        ))
14905    }
14906
14907    fn crash(at_ms: u64) -> ExitReport {
14908        ExitReport {
14909            kind: ExitKind::Crash,
14910            code: Some(1),
14911            signal: None,
14912            at_ms,
14913        }
14914    }
14915
14916    /// Record an exit on another thread and report whether it finished within
14917    /// `bound`. The recorder thread is left running if it did not.
14918    fn record_within(
14919        module_id: &'static str,
14920        ring: &Arc<Mutex<TerminalRing>>,
14921        at_ms: u64,
14922        bound: Duration,
14923    ) -> bool {
14924        let ring = Arc::clone(ring);
14925        let (done, done_rx) = std_mpsc::channel();
14926        std::thread::spawn(move || {
14927            record_terminal(
14928                module_id,
14929                &ring,
14930                &SpawnEventFeed::default(),
14931                &crash(at_ms),
14932                TerminalDisposition::Restarting,
14933            );
14934            let _ = done.send(());
14935        });
14936        done_rx.recv_timeout(bound).is_ok()
14937    }
14938
14939    /// A history read in progress must not hold the journal writer (which every
14940    /// module's exit recording needs) or the module's own ring. Exits recorded
14941    /// while the read is paused complete promptly; the paused read answers as of
14942    /// the moment it started, and the next read has each exit exactly once.
14943    #[test]
14944    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
14945        let dir = ScratchDir::new("terminal-history-concurrent-read");
14946        let path = dir.join("terminals.jsonl");
14947        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
14948            path.clone(),
14949            "daemon".into(),
14950        ));
14951        let reader_ring = journaled_ring(&journal);
14952        let other_ring = journaled_ring(&journal);
14953        assert!(record_within(
14954            "reader-module",
14955            &reader_ring,
14956            10,
14957            Duration::from_secs(5)
14958        ));
14959
14960        let (started, release) = read_pause::install(&path);
14961        let reading = {
14962            let ring = Arc::clone(&reader_ring);
14963            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
14964        };
14965        started
14966            .recv_timeout(Duration::from_secs(5))
14967            .expect("the history read reached its pause");
14968
14969        let bound = Duration::from_secs(1);
14970        assert!(
14971            record_within("other-module", &other_ring, 20, bound),
14972            "another module's exit waited on a history read (journal writer held)"
14973        );
14974        assert!(
14975            record_within("reader-module", &reader_ring, 30, bound),
14976            "the read module's own exit waited on its history read (ring held)"
14977        );
14978
14979        drop(release);
14980        let paused = reading.join().unwrap();
14981        assert_eq!(
14982            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
14983            vec![10],
14984            "an exit recorded after the read began lands in neither half of it"
14985        );
14986        assert_eq!(paused.journal_skipped_lines, 0);
14987        assert_eq!(paused.journal_read_errors, 0);
14988
14989        let after = durable_terminal_history_of(&reader_ring, "reader-module");
14990        assert_eq!(
14991            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
14992            vec![10, 30],
14993            "the next read merges ring and journal with no duplicate"
14994        );
14995        assert_eq!(after.journal_skipped_lines, 0);
14996    }
14997}
14998
14999/// What a restart does with the exited process's output readers. These drive
15000/// the same settle functions the supervisor calls, with a reader the test
15001/// holds, so a reader that has not been scheduled by the bound is a controlled
15002/// input rather than something only a loaded machine produces.
15003#[cfg(test)]
15004mod stderr_settle_tests {
15005    use std::{
15006        future::Future,
15007        io,
15008        pin::Pin,
15009        sync::{Arc, Mutex},
15010        task::{Context, Poll},
15011        time::Duration,
15012    };
15013
15014    use tokio::{
15015        io::{AsyncRead, ReadBuf},
15016        sync::oneshot,
15017        time::Instant,
15018    };
15019
15020    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
15021    use crate::stderr_tail::{
15022        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
15023        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
15024    };
15025
15026    const BOUND: Duration = Duration::from_millis(250);
15027
15028    /// Yields `before`, then stays pending until the gate is released, then
15029    /// yields `after` and reaches EOF. The bytes after the gate were written
15030    /// by a process that has already exited; only the reader is behind.
15031    struct HeldReader {
15032        before: Option<Vec<u8>>,
15033        gate: Option<oneshot::Receiver<()>>,
15034        after: io::Cursor<Vec<u8>>,
15035    }
15036
15037    impl AsyncRead for HeldReader {
15038        fn poll_read(
15039            mut self: Pin<&mut Self>,
15040            cx: &mut Context<'_>,
15041            buf: &mut ReadBuf<'_>,
15042        ) -> Poll<io::Result<()>> {
15043            if let Some(bytes) = self.before.take() {
15044                buf.put_slice(&bytes);
15045                return Poll::Ready(Ok(()));
15046            }
15047            if let Some(gate) = self.gate.as_mut() {
15048                match Pin::new(gate).poll(cx) {
15049                    Poll::Pending => return Poll::Pending,
15050                    Poll::Ready(_) => self.gate = None,
15051                }
15052            }
15053            Pin::new(&mut self.after).poll_read(cx, buf)
15054        }
15055    }
15056
15057    struct DiscardSink;
15058
15059    impl OutputSink for DiscardSink {
15060        fn write_line(&mut self, _line: &[u8]) {}
15061    }
15062
15063    fn line(text: &str) -> TailEntry {
15064        TailEntry::Line {
15065            text: text.to_string(),
15066            truncated: false,
15067            at_ms: None,
15068        }
15069    }
15070
15071    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
15072        ring.lock().unwrap()
15073    }
15074
15075    /// Start a reader for a new process generation that delivers `before`
15076    /// immediately and `after` only once the returned sender fires (or is
15077    /// dropped).
15078    fn held_pump(
15079        ring: &Arc<Mutex<StderrRing>>,
15080        before: &str,
15081        after: &str,
15082    ) -> (StderrPump, oneshot::Sender<()>) {
15083        let generation = lock(ring).begin_process();
15084        let (release, gate) = oneshot::channel();
15085        let reader = HeldReader {
15086            before: Some(before.as_bytes().to_vec()),
15087            gate: Some(gate),
15088            after: io::Cursor::new(after.as_bytes().to_vec()),
15089        };
15090        let task = tokio::spawn(pump_stderr_to(
15091            reader,
15092            Arc::clone(ring),
15093            generation,
15094            DiscardSink,
15095        ));
15096        (StderrPump { task, generation }, release)
15097    }
15098
15099    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
15100        for _ in 0..1000 {
15101            if done(&lock(ring)) {
15102                return;
15103            }
15104            tokio::time::sleep(Duration::from_millis(1)).await;
15105        }
15106        panic!(
15107            "ring never reached the expected state: {:?}",
15108            lock(ring).snapshot(None, None)
15109        );
15110    }
15111
15112    #[tokio::test(start_paused = true)]
15113    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
15114        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15115        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
15116
15117        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
15118        let before_release = lock(&ring).snapshot(None, None);
15119        assert!(
15120            matches!(before_release.capture, CaptureState::Incomplete { .. }),
15121            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
15122        );
15123
15124        // The restart: the next process starts and writes before the old
15125        // reader catches up.
15126        let next = lock(&ring).begin_process();
15127        lock(&ring).push_line_from(next, "next process booting");
15128        release.send(()).unwrap();
15129        wait_until(&ring, |ring| {
15130            ring.snapshot(None, None).capture == CaptureState::Captured
15131        })
15132        .await;
15133
15134        assert_eq!(
15135            untimed(lock(&ring).snapshot(None, None).entries),
15136            vec![
15137                line("booting"),
15138                line("config error: missing storage"),
15139                TailEntry::ProcessStart,
15140                line("next process booting"),
15141            ],
15142            "the crash's last line must survive a slow reader and stay in the crashed process's section"
15143        );
15144    }
15145
15146    #[tokio::test(start_paused = true)]
15147    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
15148    ) {
15149        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15150        // `_held` is never fired: a descendant keeps the pipe open for the
15151        // whole test.
15152        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
15153
15154        let started = Instant::now();
15155        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
15156        assert_eq!(
15157            started.elapsed(),
15158            BOUND,
15159            "the restart must wait exactly the bound for a pipe that stays open, no longer"
15160        );
15161
15162        let next = lock(&ring).begin_process();
15163        lock(&ring).push_line_from(next, "next process booting");
15164        tokio::time::sleep(Duration::from_secs(60)).await;
15165
15166        let snapshot = lock(&ring).snapshot(None, None);
15167        match &snapshot.capture {
15168            CaptureState::Incomplete { reason } => assert!(
15169                reason.contains("had not reached EOF") && reason.contains("250ms"),
15170                "the reason must say what is missing and after how long: {reason}"
15171            ),
15172            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
15173        }
15174        assert_eq!(
15175            untimed(snapshot.entries),
15176            vec![
15177                line("parent exiting"),
15178                TailEntry::ProcessStart,
15179                line("next process booting"),
15180            ]
15181        );
15182    }
15183
15184    #[tokio::test(start_paused = true)]
15185    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
15186        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15187        let (pump, release) = held_pump(&ring, "one\n", "two\n");
15188        release.send(()).unwrap();
15189
15190        settle_stderr_pump("clean", &ring, pump, BOUND).await;
15191
15192        let snapshot = lock(&ring).snapshot(None, None);
15193        assert_eq!(snapshot.capture, CaptureState::Captured);
15194        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
15195    }
15196
15197    #[tokio::test(start_paused = true)]
15198    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
15199        const BURST: usize = 150;
15200        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
15201        let path = capture.join("two-pipe.stderr.log");
15202        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
15203        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15204        let generation = lock(&ring).begin_process();
15205        let stdout = (0..BURST)
15206            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
15207            .collect::<String>();
15208        let stderr = (0..BURST)
15209            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
15210            .collect::<String>();
15211        // Model a child that has written both bursts and exited. Hold the stdout
15212        // reader after one line and a partial prefix, as if pipe readiness or
15213        // capture-file writes had delayed it. The final line also needs EOF to
15214        // flush: neither complete nor partial buffered output may be discarded.
15215        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
15216        let (release, gate) = oneshot::channel();
15217        let reader = HeldReader {
15218            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
15219            gate: Some(gate),
15220            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
15221        };
15222        let (finished, completion) = oneshot::channel();
15223        let stdout_sink = sink.clone();
15224        let pump = tokio::spawn(async move {
15225            pump_stdout_to(reader, stdout_sink).await;
15226            finished.send(()).unwrap();
15227        });
15228        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
15229
15230        let started = Instant::now();
15231        settle_stdout_pump("two-pipe", pump, BOUND).await;
15232        assert_eq!(
15233            started.elapsed(),
15234            BOUND,
15235            "a late reader must not hold up restart"
15236        );
15237        assert_eq!(
15238            std::fs::read_to_string(&path).unwrap().lines().count(),
15239            BURST + 1
15240        );
15241
15242        release
15243            .send(())
15244            .expect("stdout reader must survive the exit drain bound");
15245        tokio::time::timeout(Duration::from_secs(1), completion)
15246            .await
15247            .expect("released stdout reader must reach EOF")
15248            .expect("stdout capture must finish at EOF, not be cancelled");
15249        let contents = std::fs::read_to_string(&path).unwrap();
15250        let mut lines = contents
15251            .lines()
15252            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
15253            .collect::<Vec<_>>();
15254        let mut expected = (0..BURST)
15255            .flat_map(|i| {
15256                [
15257                    format!("out-{i:04}-{}", "o".repeat(64)),
15258                    format!("err-{i:04}-{}", "e".repeat(64)),
15259                ]
15260            })
15261            .collect::<Vec<_>>();
15262        lines.sort_unstable();
15263        expected.sort_unstable();
15264        assert_eq!(
15265            lines, expected,
15266            "both pipes must retain every complete line exactly once"
15267        );
15268    }
15269}
15270
15271/// Containment of a module's process tree (issue #109).
15272///
15273/// The behaviour these defend against is a module helper surviving its module:
15274/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
15275/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
15276/// compounds it.
15277///
15278/// They run against the SUPERVISOR rather than the job-object crate because the
15279/// claim is about teardown: a crate-level test proves a job can reap a tree, not
15280/// that the daemon's drain path reaches it.
15281///
15282/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
15283/// lane there is a separate containment path with its own tests.
15284#[cfg(all(test, windows))]
15285mod job_containment_tests {
15286    use super::*;
15287    use cortexkit_test_support::ScratchDir;
15288    use std::{
15289        path::{Path, PathBuf},
15290        sync::{Arc, Mutex},
15291        time::{Duration, Instant},
15292    };
15293
15294    /// The stub, expected beside this test executable.
15295    ///
15296    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
15297    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
15298    /// failure then reads as a broken test rather than an unbuilt dependency.
15299    fn stub_path() -> PathBuf {
15300        let mut path = std::env::current_exe().expect("current_exe available in tests");
15301        path.pop();
15302        path.pop();
15303        path.push("fake-aft-stub.exe");
15304        assert!(
15305            path.exists(),
15306            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
15307             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
15308            path.display()
15309        );
15310        path
15311    }
15312
15313    /// Poll for the grandchild pid the stub records, and parse it.
15314    fn read_grandchild_pid(path: &Path) -> u32 {
15315        let deadline = Instant::now() + Duration::from_secs(10);
15316        loop {
15317            if let Ok(contents) = std::fs::read_to_string(path) {
15318                if let Ok(pid) = contents.trim().parse() {
15319                    return pid;
15320                }
15321            }
15322            assert!(
15323                Instant::now() < deadline,
15324                "the stub never recorded a grandchild pid at {}",
15325                path.display()
15326            );
15327            std::thread::sleep(Duration::from_millis(10));
15328        }
15329    }
15330
15331    /// Everything one fixture run needs, so the two tests below differ in exactly
15332    /// one place: whether the child is contained.
15333    struct Fixture {
15334        _dir: ScratchDir,
15335        module_id: String,
15336        grandchild: u32,
15337        child: Option<SupervisedChild>,
15338        registry: Arc<Registry>,
15339        snapshot: Arc<Mutex<SupervisorSnapshot>>,
15340        terminal_ring: Arc<Mutex<TerminalRing>>,
15341        spawn_events: SpawnEventFeed,
15342    }
15343
15344    fn fixture(label: &str, module_id: &str) -> Fixture {
15345        let dir = ScratchDir::new(label);
15346        let pid_file = dir.join("grandchild.pid");
15347        let supervisor = Supervisor::new_for_test(
15348            Arc::new(Registry::default()),
15349            RestartPolicy::new(3, Duration::ZERO),
15350        );
15351        let runtime = supervisor.runtime_config();
15352        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15353        let spec = ModuleSpec {
15354            module_id: module_id.to_string(),
15355            program: stub_path(),
15356            // Zero args deliberately: a `--subc` argument would make the stub dial
15357            // a daemon that is not there, and the failure would land in the same
15358            // stderr ring this fixture exists to keep quiet.
15359            args: Vec::new(),
15360            env: vec![
15361                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
15362                (
15363                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
15364                    pid_file.display().to_string(),
15365                ),
15366            ],
15367            reserved: false,
15368            reserved_prefixes: Vec::new(),
15369            protocol: ModuleProtocol::Subc,
15370            overlap: Default::default(),
15371        };
15372        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
15373            .expect("spawn the supervised fixture");
15374        let grandchild = read_grandchild_pid(&pid_file);
15375        Fixture {
15376            _dir: dir,
15377            module_id: module_id.to_string(),
15378            grandchild,
15379            child: Some(child),
15380            registry: Arc::new(Registry::default()),
15381            snapshot,
15382            terminal_ring: Arc::clone(&runtime.terminal_ring),
15383            spawn_events: SpawnEventFeed::default(),
15384        }
15385    }
15386
15387    impl Fixture {
15388        /// Drain through the supervisor's own teardown path.
15389        async fn drain(&mut self) {
15390            let child = self
15391                .child
15392                .take()
15393                .expect("the fixture child is still present");
15394            drain_child_to_state(
15395                &self.module_id,
15396                ModuleProtocol::Subc,
15397                // No forwarding table in this fixture, so nothing reaches the
15398                // child over a connection.
15399                StopNotice::NotSent,
15400                &self.registry,
15401                None,
15402                &self.snapshot,
15403                &self.terminal_ring,
15404                &self.spawn_events,
15405                child,
15406                Duration::from_millis(500),
15407                ModuleState::Stopped,
15408                Some(false),
15409            )
15410            .await
15411            .expect("drain the supervised fixture");
15412        }
15413    }
15414
15415    /// Teardown reaps the grandchild, not merely the direct child.
15416    ///
15417    /// This is the assertion the change exists for. Before containment the
15418    /// grandchild survived: it is a separate process, and `start_kill` is
15419    /// `TerminateProcess` scoped to one pid.
15420    #[tokio::test]
15421    async fn teardown_reaps_the_grandchild() {
15422        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
15423        let grandchild = fixture.grandchild;
15424
15425        assert!(
15426            subc_jobobject::process_exists(grandchild),
15427            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
15428        );
15429
15430        fixture.drain().await;
15431
15432        assert!(
15433            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
15434            "grandchild {grandchild} outlived module teardown: the tree was not contained"
15435        );
15436    }
15437
15438    /// The mutation control: with containment withheld, the grandchild survives
15439    /// the same kill.
15440    ///
15441    /// This is the defect reproduction from #109 — a direct-child kill reaches
15442    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
15443    /// supervisor because `spawn_and_mark_running` now always contains on
15444    /// Windows, which is the point: there is no longer a path that spawns
15445    /// uncontained, so the control has to construct one.
15446    ///
15447    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
15448    /// grandchild ever dies here, that test is passing for a reason unrelated to
15449    /// the job object and the containment claim is unproven.
15450    #[test]
15451    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
15452        let dir = ScratchDir::new("teardown-uncontained");
15453        let pid_file = dir.join("grandchild.pid");
15454        let mut child = std::process::Command::new(stub_path())
15455            .env("FAKE_AFT_NEVER_CONNECT", "1")
15456            .env(
15457                "FAKE_AFT_GRANDCHILD_PID_FILE",
15458                pid_file.display().to_string(),
15459            )
15460            .stdin(std::process::Stdio::null())
15461            .stdout(std::process::Stdio::null())
15462            .stderr(std::process::Stdio::null())
15463            .spawn()
15464            .expect("spawn the uncontained fixture");
15465        let grandchild = read_grandchild_pid(&pid_file);
15466
15467        // Exactly what the pre-fix teardown did: kill the direct child.
15468        child.kill().expect("kill the direct child");
15469        let _ = child.wait();
15470
15471        assert!(
15472            subc_jobobject::process_exists(grandchild),
15473            "grandchild {grandchild} died with the direct child, so this control no longer \
15474             distinguishes contained from uncontained teardown and the regression test is \
15475             passing vacuously"
15476        );
15477
15478        // The orphan this control demonstrates is the leak the fix prevents, so
15479        // the control must not leave one behind.
15480        kill_tree(grandchild);
15481    }
15482
15483    /// Crash durability: closing the containment handle reaps the tree with no
15484    /// teardown code running at all.
15485    ///
15486    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
15487    /// call anything — and it is why containment is a kernel property of the
15488    /// handle rather than a step in the drain. Discovered by getting the
15489    /// mutation control wrong: clearing `job` to "disable" containment instead
15490    /// killed the tree, which is the guarantee, not a mistake.
15491    #[tokio::test]
15492    async fn dropping_containment_reaps_the_grandchild() {
15493        let mut fixture = fixture("drop-containment", "tree-drop");
15494        let grandchild = fixture.grandchild;
15495
15496        assert!(subc_jobobject::process_exists(grandchild));
15497
15498        // No `drain` call, no kill: dropping the handle is the entire mechanism.
15499        fixture.child.as_mut().expect("child present").job = None;
15500
15501        assert!(
15502            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
15503            "grandchild {grandchild} survived the containment handle closing, so a daemon \
15504             crash would leave the tree behind"
15505        );
15506    }
15507
15508    /// Kill a pid and its tree, then confirm it is gone.
15509    fn kill_tree(pid: u32) {
15510        let _ = std::process::Command::new("taskkill.exe")
15511            .args(["/PID", &pid.to_string(), "/T", "/F"])
15512            .stdin(std::process::Stdio::null())
15513            .stdout(std::process::Stdio::null())
15514            .stderr(std::process::Stdio::null())
15515            .status();
15516        assert!(
15517            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
15518            "could not clean up grandchild {pid}"
15519        );
15520    }
15521}
15522
15523#[cfg(test)]
15524mod privacy_trampoline_configuration_tests {
15525    #[tokio::test]
15526    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15527    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
15528        #[cfg(target_os = "macos")]
15529        {
15530            let supervisor = super::Supervisor::new(
15531                std::sync::Arc::new(crate::Registry::default()),
15532                super::RestartPolicy::default(),
15533            );
15534            let error = supervisor.spawn(spec()).unwrap_err();
15535            assert!(
15536                error
15537                    .to_string()
15538                    .contains("no privacy trampoline configured"),
15539                "{error}"
15540            );
15541        }
15542    }
15543
15544    #[tokio::test]
15545    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15546    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
15547        #[cfg(target_os = "macos")]
15548        {
15549            let supervisor = super::Supervisor::new(
15550                std::sync::Arc::new(crate::Registry::default()),
15551                super::RestartPolicy::default(),
15552            )
15553            .with_privacy_trampoline(std::env::current_exe().unwrap());
15554            let error = supervisor.spawn(spec()).unwrap_err();
15555            assert!(
15556                error
15557                    .to_string()
15558                    .contains("binary does not implement the privacy trampoline protocol"),
15559                "{error}"
15560            );
15561        }
15562    }
15563
15564    #[cfg(target_os = "macos")]
15565    fn spec() -> super::ModuleSpec {
15566        super::ModuleSpec {
15567            module_id: "privacy-configuration".into(),
15568            program: "/bin/sleep".into(),
15569            args: vec!["30".into()],
15570            env: vec![],
15571            reserved: false,
15572            reserved_prefixes: vec![],
15573            protocol: subc_control::ModuleProtocol::None,
15574            overlap: super::ModuleOverlap::Exclusive,
15575        }
15576    }
15577}
15578
15579#[cfg(test)]
15580mod privacy_exec_boundary_tests {
15581    #[cfg(target_os = "macos")]
15582    use super::*;
15583    #[cfg(target_os = "macos")]
15584    use std::{
15585        io::{Read, Write},
15586        net::{TcpListener, TcpStream},
15587    };
15588
15589    /// Unit-test-only pause at the actual early image read, not at a later
15590    /// status read. Production supervisors never inspect this environment key.
15591    #[cfg(target_os = "macos")]
15592    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
15593        if let Some((_, path)) = spec
15594            .env
15595            .iter()
15596            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
15597        {
15598            let mut barrier = TcpStream::connect(path).unwrap();
15599            barrier
15600                .set_read_timeout(Some(Duration::from_secs(30)))
15601                .unwrap();
15602            barrier.write_all(&pid.to_ne_bytes()).unwrap();
15603            let mut release = [0];
15604            barrier.read_exact(&mut release).unwrap();
15605            assert_eq!(&release, b"X");
15606        }
15607    }
15608
15609    #[cfg(target_os = "macos")]
15610    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
15611        ModuleSpec {
15612            module_id: "privacy-boundary".into(),
15613            program: program.into(),
15614            args: args.iter().map(|arg| (*arg).into()).collect(),
15615            env: vec![],
15616            reserved: false,
15617            reserved_prefixes: vec![],
15618            protocol: ModuleProtocol::None,
15619            overlap: ModuleOverlap::Exclusive,
15620        }
15621    }
15622
15623    #[cfg(target_os = "macos")]
15624    async fn accept(listener: TcpListener) -> TcpStream {
15625        // Socket readiness, not elapsed time, establishes both pause points.
15626        let listener = tokio::net::TcpListener::from_std({
15627            listener.set_nonblocking(true).unwrap();
15628            listener
15629        })
15630        .unwrap();
15631        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
15632            .await
15633            .unwrap()
15634            .unwrap();
15635        let stream = stream.into_std().unwrap();
15636        stream.set_nonblocking(false).unwrap();
15637        stream
15638            .set_read_timeout(Some(Duration::from_secs(30)))
15639            .unwrap();
15640        stream
15641    }
15642
15643    #[tokio::test]
15644    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15645    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
15646        #[cfg(target_os = "macos")]
15647        {
15648            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
15649            // Loopback sockets also work when the replay adapter's TMPDIR is
15650            // longer than Darwin's Unix-domain socket path limit.
15651            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
15652            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
15653            let record = root.join("live-children.json");
15654            let supervisor =
15655                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15656                    .with_live_children_record(&record);
15657            let runtime = supervisor.runtime_config();
15658            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15659            let mut spec = spec("/bin/sleep", &["30"]);
15660            spec.env = vec![
15661                (
15662                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
15663                    exec_listener.local_addr().unwrap().to_string(),
15664                ),
15665                (
15666                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
15667                    sample_listener.local_addr().unwrap().to_string(),
15668                ),
15669            ];
15670            let spawn = tokio::task::spawn_blocking(move || {
15671                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
15672            });
15673            let mut sample = accept(sample_listener).await;
15674            let mut pid = [0; 4];
15675            sample.read_exact(&mut pid).unwrap();
15676            let pid = u32::from_ne_bytes(pid);
15677            let mut exec = accept(exec_listener).await;
15678            let mut ready = [0];
15679            exec.read_exact(&mut ready).unwrap();
15680            assert_eq!(&ready, b"R");
15681            // The early read is guaranteed to see a real, non-null trampoline
15682            // image: the fixture has reached its barrier and cannot exec yet.
15683            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
15684            assert_eq!(
15685                observe_spawned_image(pid).unwrap().executable,
15686                Some(trampoline)
15687            );
15688            sample.write_all(b"X").unwrap();
15689            let mut child = spawn.await.unwrap();
15690            let early = crate::live_children::read_record(&record).unwrap();
15691            assert_eq!(early.len(), 1);
15692            assert_eq!(early[0].pid, pid);
15693            assert_eq!(
15694                early[0].executable, None,
15695                "unconfirmed trampoline image entered the roster"
15696            );
15697            assert!(child.report_ready.get().is_none());
15698            // The barrier's duration is unrelated to the production five-second
15699            // exec budget. Start the test's confirmation budget upon release.
15700            child.privacy_exec.as_mut().unwrap().deadline =
15701                tokio::time::Instant::now() + Duration::from_secs(30);
15702            exec.write_all(b"X").unwrap();
15703            child.confirm_privacy_exec().await;
15704            assert_eq!(child.spawn_failure, None);
15705            assert!(child.report_ready.get().is_some());
15706            let confirmed = crate::live_children::read_record(&record).unwrap();
15707            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
15708            assert_ne!(module, trampoline);
15709            assert_eq!(confirmed[0].executable, Some(module.into()));
15710            child.start_kill().unwrap();
15711            child.wait().await.unwrap();
15712            child.release_roster();
15713        }
15714    }
15715
15716    #[tokio::test]
15717    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15718    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
15719        #[cfg(target_os = "macos")]
15720        {
15721            let registry = Arc::new(Registry::default());
15722            let policy = RestartPolicy::new(0, Duration::ZERO);
15723            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
15724            let runtime = supervisor.runtime_config();
15725            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15726            let spec = spec("/bin/sh", &["-c", "exit 121"]);
15727            // Drive spawn and confirmation separately instead of starting the
15728            // monitor. WNOWAIT observes a real exit without consuming its status,
15729            // so confirmation's first try_wait must take the already-exited arm.
15730            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
15731            let pid = child.pid;
15732            tokio::task::spawn_blocking(move || {
15733                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
15734            })
15735            .await
15736            .unwrap()
15737            .unwrap();
15738            child.privacy_exec.as_mut().unwrap().deadline =
15739                tokio::time::Instant::now() + Duration::from_secs(30);
15740            let status = child.wait().await.unwrap();
15741            assert_eq!(status.code(), Some(121));
15742            assert!(child.privacy_exec.is_none());
15743            assert!(
15744                child.report_ready.get().is_none(),
15745                "an exited module must not publish a live pid"
15746            );
15747            let report = classify_reaped_child_exit(&snapshot, &child, &status);
15748            on_child_exit(
15749                &spec,
15750                policy,
15751                &registry,
15752                &snapshot,
15753                &runtime.terminal_ring,
15754                &runtime.spawn_events,
15755                &runtime.child_roster,
15756                report,
15757            )
15758            .await;
15759            let state = lock_snapshot(&snapshot).unwrap();
15760            assert_eq!(state.state, ModuleState::Failed);
15761            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
15762            assert_eq!(state.reported_pid(), None);
15763            drop(state);
15764            let history = runtime.terminal_ring.lock().unwrap().snapshot();
15765            assert_eq!(history.entries.len(), 1);
15766            let terminal = &history.entries[0];
15767            assert_eq!(terminal.exit_code, Some(121));
15768            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
15769            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
15770            assert_eq!(
15771                terminal.disposition_detail.as_deref(),
15772                Some(policy.budget_exhausted_detail().as_str()),
15773                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
15774            );
15775            assert_eq!(child.spawn_failure, None);
15776            child.release_roster();
15777        }
15778    }
15779}
15780
15781/// The daemon's real spawn path hands a subc-wire child its launch nonce on
15782/// descriptor 3, without an environment copy. The shell records the nonce
15783/// and its environment after exec so these tests observe the real handover.
15784#[cfg(all(test, unix))]
15785mod launch_nonce_descriptor_tests {
15786    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
15787    use crate::stderr_tail::{StderrRing, StderrTailConfig};
15788    use cortexkit_test_support::ScratchDir;
15789    use std::{
15790        path::PathBuf,
15791        sync::{Arc, Mutex},
15792        time::{Duration, Instant},
15793    };
15794
15795    async fn probe(role: super::SpawnRole) {
15796        let scratch = ScratchDir::new("launch-nonce-descriptor");
15797        let fd_copy = scratch.join("from-descriptor");
15798        let env_copy = scratch.join("environment");
15799        let script = format!(
15800            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
15801            fd = fd_copy.display(), env = env_copy.display(),
15802        );
15803        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
15804        let spec = ModuleSpec {
15805            module_id: "nonce-descriptor-probe".to_string(),
15806            program: PathBuf::from("/bin/sh"),
15807            args: vec!["-c".to_string(), script],
15808            env: vec![
15809                xdg("XDG_DATA_HOME"),
15810                xdg("XDG_RUNTIME_DIR"),
15811                xdg("XDG_CONFIG_HOME"),
15812                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
15813            ],
15814            reserved: true,
15815            reserved_prefixes: Vec::new(),
15816            protocol: ModuleProtocol::Subc,
15817            overlap: Default::default(),
15818        };
15819        let handle = SupervisorHandle::new();
15820        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15821        let roster = ChildRoster::default();
15822        #[cfg(target_os = "macos")]
15823        {
15824            let path = super::test_privacy_trampoline();
15825            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
15826        }
15827        let child = super::spawn_child_in_slot(
15828            &spec,
15829            None,
15830            Some(&handle),
15831            &ring,
15832            None,
15833            &roster,
15834            #[cfg(target_os = "linux")]
15835            None,
15836            role,
15837            matches!(role, super::SpawnRole::SwapCandidate),
15838        )
15839        .expect("spawn probe");
15840        let deadline = Instant::now() + Duration::from_secs(10);
15841        while !(fd_copy.exists() && env_copy.exists()) {
15842            assert!(Instant::now() < deadline, "probe never wrote its copies");
15843            tokio::time::sleep(Duration::from_millis(20)).await;
15844        }
15845        let nonce = std::fs::read_to_string(fd_copy).unwrap();
15846        assert!(!nonce.is_empty());
15847        let environment = std::fs::read_to_string(env_copy).unwrap();
15848        assert!(environment
15849            .lines()
15850            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
15851        let copy = environment
15852            .lines()
15853            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
15854        assert_eq!(
15855            copy, None,
15856            "Unix children must never receive the environment nonce"
15857        );
15858        if matches!(role, super::SpawnRole::Plain) {
15859            assert_eq!(
15860                handle.spawn_nonce(&spec.module_id).as_deref(),
15861                Some(nonce.as_str())
15862            );
15863        }
15864        drop(child);
15865    }
15866
15867    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15868    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
15869        probe(super::SpawnRole::Plain).await;
15870    }
15871
15872    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15873    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
15874        probe(super::SpawnRole::SwapCandidate).await;
15875    }
15876}
15877
15878#[cfg(all(test, target_os = "linux"))]
15879mod cgroup_containment_tests {
15880    use super::*;
15881    use cortexkit_test_support::ScratchDir;
15882
15883    fn running(pid: u32) -> bool {
15884        // An orphan can remain a zombie until the container init reaps it.
15885        std::fs::read_to_string(format!("/proc/{pid}/stat"))
15886            .ok()
15887            .and_then(|stat| {
15888                stat.rsplit_once(") ")
15889                    .map(|(_, rest)| rest.starts_with('Z'))
15890            })
15891            .is_some_and(|zombie| !zombie)
15892    }
15893
15894    #[tokio::test]
15895    async fn linux_teardown_reaps_the_grandchild() {
15896        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
15897    }
15898
15899    #[tokio::test]
15900    async fn linux_shutdown_straggler_reaps_the_grandchild() {
15901        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
15902    }
15903
15904    async fn teardown_tree(test_name: &str, shutdown: bool) {
15905        let dir = ScratchDir::new(test_name);
15906        let root = PathBuf::from(format!(
15907            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
15908            std::process::id(),
15909            unix_ms_now()
15910        ));
15911        if let Err(error) = std::fs::create_dir(&root) {
15912            assert!(
15913                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15914                "required cgroup test cannot execute: {error}"
15915            );
15916            eprintln!(
15917                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
15918                root.display()
15919            );
15920            return;
15921        }
15922        let placement = subc_cgroup::prepare_at(&root)
15923            .expect("prepare isolated kernel cgroup")
15924            .expect("isolated cgroup is delegated");
15925        let module_id = "tree-teardown";
15926        let module = placement
15927            .module_path(module_id)
15928            .expect("create isolated module cgroup");
15929        if !module.join("cgroup.kill").exists() {
15930            std::fs::remove_dir(&module).unwrap();
15931            std::fs::remove_dir(root.join("subc-modules")).unwrap();
15932            std::fs::remove_dir(&root).unwrap();
15933            assert!(
15934                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15935                "required cgroup.kill interface unavailable"
15936            );
15937            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
15938            return;
15939        }
15940        let supervisor = Supervisor::new_for_test(
15941            Arc::new(Registry::default()),
15942            RestartPolicy::new(3, Duration::ZERO),
15943        )
15944        .with_cgroup_placement(Some(placement));
15945        let mut runtime = supervisor.runtime_config();
15946        runtime.child_roster = runtime
15947            .child_roster
15948            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
15949        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15950        let pid_file = dir.join("grandchild.pid");
15951        let spec = ModuleSpec {
15952            module_id: module_id.to_string(),
15953            program: PathBuf::from("/bin/sh"),
15954            args: vec![
15955                "-c".into(),
15956                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
15957                "fixture".into(),
15958                pid_file.display().to_string(),
15959            ],
15960            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15961                .into_iter()
15962                .map(|key| (key.to_string(), dir.display().to_string()))
15963                .collect(),
15964            reserved: false,
15965            reserved_prefixes: Vec::new(),
15966            protocol: ModuleProtocol::None,
15967            overlap: Default::default(),
15968        };
15969        let child =
15970            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
15971        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
15972        let grandchild: u32 = loop {
15973            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
15974                if let Ok(pid) = contents.trim().parse() {
15975                    break pid;
15976                }
15977            }
15978            assert!(
15979                tokio::time::Instant::now() < deadline,
15980                "grandchild pid was not recorded"
15981            );
15982            tokio::time::sleep(Duration::from_millis(10)).await;
15983        };
15984        assert!(
15985            running(grandchild),
15986            "grandchild must be alive before teardown"
15987        );
15988        if shutdown {
15989            let mut child = child;
15990            crate::child_roster::end_children_for_daemon_shutdown(
15991                &runtime.child_roster,
15992                false,
15993                std::future::pending(),
15994            )
15995            .await;
15996            child.wait().await.expect("reap shutdown straggler");
15997        } else {
15998            drain_child_to_state(
15999                module_id,
16000                ModuleProtocol::None,
16001                StopNotice::NotSent,
16002                &Registry::default(),
16003                None,
16004                &snapshot,
16005                &runtime.terminal_ring,
16006                &SpawnEventFeed::default(),
16007                child,
16008                Duration::from_millis(100),
16009                ModuleState::Stopped,
16010                Some(false),
16011            )
16012            .await
16013            .expect("real supervisor teardown");
16014        }
16015        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16016        while running(grandchild) && tokio::time::Instant::now() < deadline {
16017            tokio::time::sleep(Duration::from_millis(10)).await;
16018        }
16019        let survived = running(grandchild);
16020        // Kill a surviving grandchild so a failed test does not leave it behind.
16021        if survived {
16022            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
16023            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
16024            tokio::time::sleep(Duration::from_millis(100)).await;
16025        }
16026        if module.exists() {
16027            std::fs::remove_dir(&module).expect("remove empty module cgroup");
16028        }
16029        std::fs::remove_dir(root.join("subc-modules")).unwrap();
16030        std::fs::remove_dir(&root).unwrap();
16031        assert!(
16032            !survived,
16033            "grandchild {grandchild} outlived module teardown"
16034        );
16035        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
16036    }
16037}