Skip to main content

subc_daemon/
supervise.rs

1//! Identity policy updates read the running-protocol snapshot before taking the
2//! nonce locks. Nested identity locks are taken in this order: spawn nonces,
3//! reserved nonces, then `SupervisorHandle::implicit_protocol_none`.
4
5#[cfg(all(test, target_os = "linux"))]
6#[path = "supervise/audit_identity_tests.rs"]
7mod audit_identity_tests;
8mod resource_launch;
9pub(crate) use resource_launch::{ResourceCommitError, ResourceLaunch};
10
11use std::{
12    collections::{HashMap, HashSet, VecDeque},
13    error::Error,
14    fmt, io,
15    path::PathBuf,
16    process::{ExitStatus, Stdio},
17    sync::{Arc, Mutex, OnceLock},
18    time::{Duration, SystemTime, UNIX_EPOCH},
19};
20
21use cortexkit_log::Retention;
22use serde_json::Value;
23use sha2::{Digest, Sha256};
24use subc_control::{
25    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
26    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
27};
28use subc_protocol::{
29    manifest::{SelfSignalKind, SignalAnchor},
30    session::{
31        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
32        MODULE_CONTROL_OP_HEALTH_CHECK,
33    },
34    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
35};
36use tokio::{
37    process::{Child, Command},
38    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
39    task::JoinHandle,
40    time::{sleep, sleep_until, timeout, timeout_at, Instant},
41};
42use tracing::{debug, error, info, warn};
43
44use crate::{
45    child_roster::ChildRoster,
46    daemon_config::{
47        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
48    },
49    forwarding::{
50        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
51        ModuleDrainTarget, PendingModuleControlRpc,
52    },
53    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
54    registry::{ConnectionId, RegistrationEndReason, RegistryError},
55    stderr_tail::{
56        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
57        StderrTailSnapshot,
58    },
59    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
60    Frame, FrameSink, Registry,
61};
62
63#[cfg(not(windows))]
64use crate::provenance::spawned_file_identity;
65
66#[path = "supervise_swap.rs"]
67mod swap;
68
69#[path = "supervise_activity.rs"]
70mod activity;
71
72#[cfg(unix)]
73#[path = "supervise/admission.rs"]
74mod admission;
75mod admission_exit;
76#[cfg(all(test, target_os = "linux"))]
77mod async_spawn_tests;
78#[cfg(all(test, windows))]
79mod carried_job_tests;
80mod config_delivery;
81mod content_binding;
82mod program_binding;
83mod registration;
84pub(crate) mod script;
85/// Each launch nonce, mapped to its launch's module id and the launch.
86type NonceLaunches = HashMap<String, (String, std::sync::Weak<RegistrationSpawn>)>;
87pub(crate) use registration::NonceAttestation;
88#[cfg(all(test, unix))]
89pub(crate) use registration::RegistrationProof;
90pub(crate) use registration::RegistrationSpawn;
91pub(crate) use registration::REGISTRATION_CHANNEL_CLOSED;
92#[cfg(all(test, unix))]
93pub(crate) mod unix_channel_placement_tests;
94use program_binding::ProgramBindingPublication;
95mod config_exit_failure;
96#[cfg(all(test, any(target_os = "linux", windows)))]
97mod missing_path_tests;
98#[cfg(all(test, unix))]
99mod program_binding_tests;
100#[cfg(all(test, windows))]
101mod program_binding_windows_tests;
102#[cfg(all(test, windows))]
103mod windows_channel_tests;
104pub use config_exit_failure::ModuleStopReason;
105#[cfg(all(test, unix))]
106mod config_exit_hold_tests;
107pub(crate) mod containment;
108#[cfg(all(test, unix))]
109mod containment_tests;
110mod slow_retry;
111use slow_retry::{FailedRetry, FailureContext};
112mod host_health;
113use host_health::HostHealth;
114
115#[cfg(all(test, unix))]
116mod mark_readiness_tests;
117#[cfg(all(test, unix))]
118mod slow_retry_tests;
119
120/// Command-line flag used by supervised modules to find subc.
121///
122/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
123/// The path points at the TCP+key connection file; it is not an ambient signal and
124/// is never inherited by standalone children.
125pub const SUBC_ARG: &str = "--subc";
126
127const DEFAULT_MAX_RESTARTS: u32 = 3;
128const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
129const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
130/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
131/// to contain a real crash loop (which respawns in seconds) and short enough
132/// that unrelated crashes hours apart never accumulate into a permanent stop.
133const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
134/// How long a drain waits for already-dispatched requests to finalize before
135/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
136/// not RPC-scale: the original 2s value silently cut nearly every real tool
137/// call at the fence, making the wait-for-finalize design decorative for the
138/// workloads it existed for. Quiescence short-circuits, so an idle module
139/// restarts immediately regardless of this value; the budget is spent only
140/// when a genuine in-flight request is worth finishing. Per-module override:
141/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
142/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
143/// where a stuck request will never settle).
144pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
145pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
146const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
147const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
148/// How long a drain waits after one busy-gauge probe completes before it asks
149/// again.
150///
151/// A drain re-probes on this short interval of its own rather than on the
152/// module's health cadence. The default cadence (30 s) equals the default
153/// drain ceiling, so a cadence-paced drain that found the module busy at its
154/// first probe never asked a second time: it waited out the whole ceiling and
155/// forced GOODBYE even after the gauges had read zero. The interval is counted
156/// from the moment the previous probe finished, and probes run one at a time,
157/// so a slow module is never asked more often than it can answer. Periodic
158/// health checks outside a drain keep using the configured cadence.
159const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
160/// How long a restart waits for the exited process's output readers.
161///
162/// The restart does not depend on either output reader finishing. A stderr
163/// reader still running at this bound is left running, and its later output goes
164/// into the exited process's own section of the stderr ring (see
165/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
166/// bound no longer decides whether a crash's last lines are kept: under load
167/// the reader may simply not have been scheduled yet, and cutting it there
168/// discarded exactly the lines that explained the crash.
169///
170/// What the bound still decides is when the tail starts reporting
171/// `Incomplete`: a pipe open past it usually means a descendant of the exited
172/// process still holds it, and the tail cannot claim to be whole until that
173/// pipe closes. That is also the only case in which the wait costs the restart
174/// anything, because a pipe with no other holder reaches EOF when the process
175/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
176/// `Captured` at EOF with nothing lost.
177///
178/// The stdout reader carries no ring, only the capture file. It also continues
179/// to EOF after this bound: child exit closes the writer but does not mean the
180/// reader has consumed all buffered bytes. Late output from either pipe can
181/// therefore reach the shared capture file after the successor starts.
182const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
183/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
184pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
185const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
186/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
187/// dropped because the subscriber stopped draining its frames. The detail's
188/// `first_undelivered_cursor` names the first event it did not receive; the
189/// client resubscribes from the last cursor it did receive.
190pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
191
192#[cfg(target_os = "macos")]
193struct PrivacyExec {
194    content_check: Option<tokio::task::JoinHandle<Result<(), crate::config_gate::GateError>>>,
195    gated: bool,
196    admission_binding: Option<ProgramBindingPublication>,
197    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
198    deadline: tokio::time::Instant,
199    expected: Option<subc_os::FileIdentity>,
200    trampoline: Option<subc_os::FileIdentity>,
201    signature: Option<Arc<subc_os::privacy_identity::SuspendedSignature>>,
202    module_id: String,
203    #[cfg(test)]
204    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
205    #[cfg(test)]
206    image_probe: Option<fn(u32) -> Option<subc_os::Observation>>,
207    #[cfg(test)]
208    image_read_delay: Option<Duration>,
209}
210
211#[cfg(all(test, target_os = "macos"))]
212pub(crate) fn test_privacy_trampoline() -> PathBuf {
213    // The fixture bin lives in the profile folder the unit-test executable was
214    // built into. This honors custom CARGO_TARGET_DIR too.
215    let path = cortexkit_test_support::cargo_profile_bin("privacy-trampoline-fixture");
216    // Without the fixture every macOS spawn is refused, and the tests that
217    // spawn fail later as a module in state Failed, which names the wrong
218    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
219    assert!(
220        path.exists(),
221        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
222         --bins --features test-support` or `cargo test -p subc-daemon` first",
223        path.display()
224    );
225    path
226}
227
228#[cfg(target_os = "macos")]
229fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
230    use std::io::Read;
231    let mut probe = subc_os::child_spawn::spawn(
232        std::process::Command::new(path)
233            .args(["__disclaim-exec", "--probe-suspended"])
234            .stdin(Stdio::null())
235            .stdout(Stdio::piped())
236            .stderr(Stdio::piped()),
237    )
238    .map_err(|error| {
239        format!(
240            "privacy trampoline probe failed for {}: {error}",
241            path.display()
242        )
243    })?;
244    let deadline = std::time::Instant::now() + Duration::from_secs(5);
245    let status = loop {
246        match probe.try_wait() {
247            Ok(Some(status)) => break status,
248            Ok(None) if std::time::Instant::now() < deadline => {
249                std::thread::sleep(Duration::from_millis(5))
250            }
251            result => {
252                let _ = probe.kill();
253                let _ = probe.wait();
254                return Err(format!(
255                    "privacy trampoline probe failed or timed out for {}: {result:?}",
256                    path.display()
257                ));
258            }
259        }
260    };
261    let mut answer = String::new();
262    if let Some(stdout) = probe.stdout.take() {
263        let _ = stdout.take(256).read_to_string(&mut answer);
264    }
265    if status.success() && answer.trim() == subc_os::privacy_identity::SUSPENDED_TRAMPOLINE_PROBE {
266        return Ok(());
267    }
268    let mut diagnostic = String::new();
269    if let Some(stderr) = probe.stderr.take() {
270        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
271    }
272    let cause = diagnostic
273        .trim()
274        .strip_prefix("ck-subc: own privacy identity refused: ")
275        .unwrap_or("binary does not implement the privacy trampoline protocol");
276    Err(format!(
277        "{cause}: probe of {} exited {status}",
278        path.display()
279    ))
280}
281
282#[cfg(target_os = "macos")]
283fn privacy_command(
284    spec: &ModuleSpec,
285    binding: &SpawnProgramBinding,
286    roster: &ChildRoster,
287    gated: bool,
288) -> Result<
289    (
290        Command,
291        Option<PrivacyExec>,
292        subc_os::privacy_identity::ExecAcknowledgement,
293    ),
294    SuperviseError,
295> {
296    let failure = |cause: String| {
297        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
298        SuperviseError::Spawn {
299            program: spec.program.clone(),
300            source: io::Error::other(cause),
301            cgroup_path: None,
302        }
303    };
304    let trampoline = roster.privacy_trampoline().map_err(failure)?;
305    let expected = Some(binding.identity);
306    let trampoline_image = subc_os::file_identity(&trampoline);
307    let signature = gated
308        .then(|| {
309            subc_os::privacy_identity::SuspendedSignature::capture(&binding._file, &binding.path)
310                .map(Arc::new)
311        })
312        .transpose()
313        .map_err(|error| failure(error.to_string()))?;
314    if expected == trampoline_image {
315        return Err(failure(
316            "privacy identity module executable is missing or is the trampoline itself".to_string(),
317        ));
318    }
319    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
320        .map_err(|error| failure(error.to_string()))?;
321    let reader =
322        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
323    let mut command = Command::new(&trampoline);
324    command
325        .arg(if gated {
326            "__disclaim-exec-suspended"
327        } else {
328            "__disclaim-exec"
329        })
330        .arg(ack.fd().to_string())
331        .arg(&binding.path);
332    Ok((
333        command,
334        Some(PrivacyExec {
335            gated,
336            admission_binding: None,
337            content_check: None,
338            reader,
339            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
340            expected,
341            trampoline: trampoline_image,
342            signature,
343            module_id: spec.module_id.clone(),
344            #[cfg(test)]
345            image_probe: None,
346            #[cfg(test)]
347            image_read_delay: spec
348                .env
349                .iter()
350                .find(|(key, _)| key == "SUBC_TEST_PRIVACY_IMAGE_DELAY_MS")
351                .map(|(_, millis)| Duration::from_millis(millis.parse().unwrap())),
352            #[cfg(test)]
353            confirmation_pause: spec
354                .env
355                .iter()
356                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
357                .map(|(_, address)| (address.clone(), None)),
358        }),
359        ack,
360    ))
361}
362
363#[cfg(unix)]
364type SpawnRegistrationChannel = std::os::unix::net::UnixStream;
365#[cfg(windows)]
366type SpawnRegistrationChannel = subc_os::registration_channel::RegistrationPipe;
367#[cfg(test)]
368type CapturedRegistrationChannels =
369    Arc<Mutex<Option<Vec<(Arc<RegistrationSpawn>, CapturedRegistrationEndpoint)>>>>;
370#[cfg(test)]
371struct CapturedRegistrationEndpoint {
372    _channel: SpawnRegistrationChannel,
373}
374#[cfg(test)]
375impl fmt::Debug for CapturedRegistrationEndpoint {
376    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
377        f.write_str("CapturedRegistrationEndpoint")
378    }
379}
380
381struct SupervisedChild {
382    registration_spawn: Option<Arc<RegistrationSpawn>>,
383    /// Owned daemon endpoint, retained for the per-spawn registration service.
384    #[cfg(any(unix, windows))]
385    registration_channel: Option<SpawnRegistrationChannel>,
386    live_program_binding: Option<Arc<content_binding::LiveProgramBinding>>,
387    registered_config_connection: Option<ConnectionId>,
388    admission_base: Option<subc_config::Digest>,
389    registration_declaration: Option<String>,
390    _program_binding: Option<ProgramBindingPublication>,
391    subtree_target: crate::subtree_resources::Target,
392    child: containment::ProcessChild,
393    exec: crate::activity::ExecOwner,
394    #[cfg(windows)]
395    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
396    #[cfg(target_os = "macos")]
397    privacy_exec: Option<PrivacyExec>,
398    /// Set once this launch's exec acknowledgement confirms the module image.
399    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
400    /// `subc_os::privacy_identity`), which then replaces itself with the
401    /// module. The supervisor owns and can kill that pid from spawn, but
402    /// status readers report it only after this latch is set, so nothing
403    /// reports the trampoline's image as the module's.
404    #[cfg(target_os = "macos")]
405    report_ready: Arc<OnceLock<()>>,
406    /// Supervisor failure detail retained for the terminal record, including
407    /// image refusals and first-HELLO timeouts after a successful spawn.
408    spawn_failure: Option<String>,
409    /// Set only after a kill requested because launch admission failed was
410    /// accepted, never for an operator, health, or shutdown kill.
411    admission_kill_sent: bool,
412    /// The protocol this process was launched with. A reload can store a new
413    /// launch spec with a different protocol, but that takes effect only at the
414    /// next spawn, so this process keeps being handled by the protocol it
415    /// actually speaks.
416    protocol: ModuleProtocol,
417    /// This container's cgroup name: a bounded module/slot label followed by a
418    /// unique suffix (when cgroup placement is on). Carried replacements reuse
419    /// the name; ordinary launches each get a fresh container. A
420    /// retired process in a slot may still be draining when a later one is
421    /// spawned into that slot, so the suffix keeps the later process out of
422    /// the retired one's cgroup, which is the domain a kill applies to.
423    #[cfg(target_os = "linux")]
424    module_id: String,
425    #[cfg(target_os = "linux")]
426    cgroup_placement: Option<subc_cgroup::Placement>,
427    /// The job for an ordinary, non-carried launch. For carried
428    /// launches the actor owns it and the child holds only a weak container link.
429    ///
430    /// Dropping this handle is what reaps a surviving tree when no supervisor
431    /// code runs — a daemon crash — because the job carries
432    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
433    ///
434    /// That limit is not crash-only, and the difference is worth knowing: a
435    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
436    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
437    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
438    /// module at once. Before this change they survived that, saw EOF on the
439    /// control socket, and ran their own teardown; Unix keeps that path
440    /// deliberately, so a module can seal a WAL or close a capture rather than
441    /// be killed mid-write. So this trades graceful teardown on every Windows
442    /// daemon stop for containment on a crash, which is the right way round
443    /// today: orphaned GPU workers are a reported, recurring problem, and the
444    /// modules that write most heavily do not run on Windows.
445    ///
446    /// The fix is a real Windows stop path — the daemon draining before it
447    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
448    /// reaches only what the drain left behind, which is what it should reach.
449    #[cfg(windows)]
450    job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
451    stdout_pump: Option<JoinHandle<()>>,
452    stderr_pump: Option<StderrPump>,
453    stderr_ring: Arc<Mutex<StderrRing>>,
454    /// Conservative monotonic bound sampled before `command.spawn()`, not when
455    /// identity recording finishes or the spawn generation is published.
456    spawned_at: Instant,
457    config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
458    /// Ordinary spawns must send their first HELLO within this budget. Once
459    /// accepted, a later disconnect must not restart the registration clock.
460    registration_deadline: Option<Instant>,
461    registration_watch: Option<RegistrationWatch>,
462    spawned_at_ms: u64,
463    spawned_from: PathBuf,
464    launch_spec_digest: LaunchSpecDigest,
465    spawned_file_identity: Option<SpawnedFileIdentity>,
466    process_start_time: Option<u64>,
467    process_identity: Option<ProcessIdentity>,
468    pid: u32,
469    /// This process's entry in the daemon's child roster, released when the
470    /// process is reaped or this handle is dropped.
471    roster_guard: Option<crate::child_roster::RosterGuard>,
472}
473
474impl SupervisedChild {
475    fn retire_registration(&self) {
476        // All exit observers enter here before cleanup can await. The retired
477        // program-binding flag and the registration's Dead state make repeated
478        // observations harmless, even when another task observes the group
479        // leader retained unreaped to reserve its process-group id.
480        if let Some(spawn) = &self.registration_spawn {
481            spawn.retire();
482        } else if let Some(binding) = &self._program_binding {
483            binding.retire();
484        }
485        #[cfg(unix)]
486        if let Some(channel) = &self.registration_channel {
487            let _ = channel.shutdown(std::net::Shutdown::Both);
488        }
489        #[cfg(windows)]
490        if let Some(channel) = &self.registration_channel {
491            if let Ok(shutdown) = channel.shutdown_handle() {
492                shutdown.shutdown();
493            }
494        }
495    }
496    fn registered_since_spawn(&self) -> bool {
497        self.registration_watch
498            .as_ref()
499            .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
500    }
501
502    fn awaiting_registration(&self) -> bool {
503        // Admission failure retires the channel before its kill becomes
504        // waitable. That closure must not replace the admission failure with
505        // a registration failure and spend crash restart budget.
506        let admission_failed = self.admission_kill_sent && self.spawn_failure.is_some();
507        !admission_failed && !self.registered_since_spawn()
508    }
509
510    fn id(&self) -> Option<u32> {
511        Some(self.pid)
512    }
513
514    fn process_identity(&self) -> Option<ProcessIdentity> {
515        self.process_identity
516    }
517
518    fn try_wait(&mut self) -> io::Result<Option<ExitStatus>> {
519        let result = self.child.try_wait();
520        if !matches!(result, Ok(None)) {
521            self.retire_registration();
522            #[cfg(any(unix, windows))]
523            self.registration_channel.take();
524            if let Some(binding) = &self._program_binding {
525                binding.retire();
526            }
527        }
528        result
529    }
530
531    async fn wait(&mut self) -> io::Result<ExitStatus> {
532        #[cfg(target_os = "macos")]
533        self.confirm_privacy_exec().await;
534        // The roster entry is NOT released here. A daemon shutdown waits for the
535        // roster to empty and then exits the process, so releasing at the reap
536        // let it exit before the exit handler wrote this child's terminal record
537        // (the stderr drain and snapshot update sit in between), and the
538        // shutdown's own `daemon_shutdown` record was intermittently lost. The
539        // caller releases it after recording the exit (`release_roster`), and
540        // dropping the handle releases it too.
541        let result = self.child.wait().await;
542        self.retire_registration();
543        #[cfg(any(unix, windows))]
544        self.registration_channel.take();
545        // All plain and swap waits pass here. Revoke the reaped child's nonce
546        // before cgroup cleanup, stderr draining or exit observation can await.
547        if let Some(binding) = &self._program_binding {
548            binding.retire();
549        }
550        #[cfg(all(test, target_os = "linux"))]
551        if let Some(binding) = &self._program_binding {
552            binding.pause_after_wait_for_test().await;
553        }
554        #[cfg(target_os = "linux")]
555        if result.is_ok() && self.child.container().is_none() {
556            if let Some(placement) = self.cgroup_placement.as_ref() {
557                // Ordinary children still own a per-spawn cgroup: returning
558                // from wait must leave it empty and removed, as before carry.
559                // Keep cleanup ownership across cancellation until it finishes.
560                cleanup_reaped_cgroup(placement, &self.module_id).await;
561                self.cgroup_placement = None;
562            }
563        }
564        result
565    }
566
567    async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
568        self.retire_registration();
569        // Idempotent fallback for cleanup paths that did not call wait.
570        if let Some(binding) = self._program_binding.take() {
571            binding.retire();
572        }
573        if self.child.container().is_some() {
574            if !carry {
575                self.child.terminate_container().await;
576            }
577        } else {
578            #[cfg(target_os = "linux")]
579            if let Some(placement) = self.cgroup_placement.as_ref() {
580                cleanup_reaped_cgroup(placement, &self.module_id).await;
581                self.cgroup_placement = None;
582            }
583        }
584        self.child.reap().await?;
585        self.release_roster();
586        Ok(())
587    }
588
589    #[cfg(target_os = "macos")]
590    async fn confirm_privacy_exec(&mut self) {
591        if self
592            .privacy_exec
593            .as_ref()
594            .is_some_and(|pending| pending.content_check.is_some())
595        {
596            self.finish_privacy_content_check().await;
597            return;
598        }
599        let Some(pending) = &mut self.privacy_exec else {
600            return;
601        };
602        let result = tokio::time::timeout_at(pending.deadline, async {
603            let mut record = Vec::new();
604            loop {
605                let mut ready = pending.reader.readable().await?;
606                let read = ready.try_io(|reader| {
607                    use std::io::Read;
608                    let mut reader = reader.get_ref();
609                    let mut buffer = [0u8; 256];
610                    reader.read(&mut buffer).map(|count| (count, buffer))
611                });
612                match read {
613                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
614                    Ok(Ok((count, buffer))) => {
615                        if record.len() + count > 1024 {
616                            return Err(io::Error::other(
617                                "privacy exec refusal record is too long",
618                            ));
619                        }
620                        record.extend_from_slice(&buffer[..count]);
621                    }
622                    Ok(Err(error)) => return Err(error),
623                    Err(_) => continue,
624                }
625            }
626        })
627        .await;
628        #[cfg(test)]
629        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
630            use tokio::io::{AsyncReadExt, AsyncWriteExt};
631            if stream.is_none() {
632                *stream = Some(
633                    tokio::net::TcpStream::connect(address.as_str())
634                        .await
635                        .unwrap(),
636                );
637                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
638                stream
639                    .as_mut()
640                    .unwrap()
641                    .write_all(&self.pid.to_ne_bytes())
642                    .await
643                    .unwrap();
644            }
645            let mut reply = [0];
646            stream
647                .as_mut()
648                .unwrap()
649                .read_exact(&mut reply)
650                .await
651                .unwrap();
652            pending.confirmation_pause = None;
653            if &reply == b"F" {
654                Ok(Err(io::Error::other("injected late exec refusal")))
655            } else {
656                assert_eq!(&reply, b"X");
657                result
658            }
659        } else {
660            result
661        };
662        // Keep the reader in self across await: select cancellation must not
663        // discard the handshake or reset its original five-second deadline.
664        // Use try_wait so a child reaped during confirmation has its program
665        // binding retired before another command can use its nonce.
666        let waited = if matches!(&result, Ok(Ok(record)) if record.is_empty()) {
667            Some(self.try_wait())
668        } else {
669            None
670        };
671        let pending = self.privacy_exec.as_ref().expect("pending exec");
672        let cause = match result {
673            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
674            Ok(Err(error)) => Some(format!(
675                "privacy identity exec acknowledgement failed: {error}"
676            )),
677            Ok(Ok(record)) if !record.is_empty() => Some(
678                std::str::from_utf8(&record)
679                    .ok()
680                    .and_then(|record| {
681                        record
682                            .trim()
683                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
684                    })
685                    .filter(|cause| !cause.is_empty())
686                    .unwrap_or("invalid privacy exec refusal record")
687                    .to_string(),
688            ),
689            Ok(Ok(_)) if !pending.gated => match waited.expect("exec EOF checked child") {
690                Ok(Some(_)) => None,
691                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
692                Ok(None) => {
693                    #[cfg(test)]
694                    if let Some(delay) = pending.image_read_delay {
695                        tokio::time::sleep(delay).await;
696                    }
697                    let image = observe_spawned_image(self.pid);
698                    #[cfg(test)]
699                    let image = pending.image_probe.map_or(image, |probe| probe(self.pid));
700                    // An unstamped module can re-exec before we wake to read its
701                    // image. Only the trampoline image proves exec never happened;
702                    // retain the observed image for later orphan identification.
703                    if let Some(image) = image.filter(|image| {
704                        image.executable.is_some() && image.executable != pending.trampoline
705                    }) {
706                        match self
707                            .roster_guard
708                            .as_ref()
709                            .map(|guard| guard.confirm_executable(image))
710                            .transpose()
711                        {
712                            Err(error) => Some(format!(
713                                "could not durably record confirmed module image: {error}"
714                            )),
715                            Ok(_) => {
716                                let _ = self.report_ready.set(());
717                                None
718                            }
719                        }
720                    } else if image.is_none()
721                        || image.is_some_and(|image| image.executable.is_none())
722                    {
723                        // Empty EOF acknowledged exec. An image disappearing
724                        // during observation is an ordinary module exit.
725                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
726                            Ok(result) => {
727                                self.retire_registration();
728                                self.registration_channel.take();
729                                if let Some(binding) = &self._program_binding {
730                                    binding.retire();
731                                }
732                                result.err().map(|error| format!("privacy identity trampoline wait failed: {error}"))
733                            }
734                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
735                        }
736                    } else {
737                        Some("privacy identity trampoline did not exec the module".to_string())
738                    }
739                }
740            },
741            Ok(Ok(_)) => match waited.expect("exec EOF checked child") {
742                Ok(Some(_)) => {
743                    Some("privacy identity target exited before suspended admission".to_string())
744                }
745                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
746                Ok(None) => {
747                    // CLOEXEC closes the ack just before the kernel reports the
748                    // suspended target. EOF alone must never admit a running image.
749                    let stopped = tokio::time::timeout_at(pending.deadline, async {
750                        loop {
751                            if subc_os::privacy_identity::SuspendedSignature::is_suspended(
752                                self.pid,
753                            )? {
754                                return Ok::<_, io::Error>(());
755                            }
756                            tokio::time::sleep(Duration::from_millis(1)).await;
757                        }
758                    })
759                    .await;
760                    match stopped {
761                        Err(_) => {
762                            Some("privacy identity target was not suspended within 5s".to_string())
763                        }
764                        Ok(Err(error)) => Some(format!(
765                            "privacy identity suspended target unavailable: {error}"
766                        )),
767                        Ok(Ok(())) => {
768                            let image = observe_spawned_image(self.pid);
769                            if image.is_some_and(|image| {
770                                image.executable.is_some()
771                                    && image.executable != pending.trampoline
772                                    && image.executable == pending.expected
773                            }) {
774                                None
775                            } else {
776                                Some("privacy identity trampoline executable mismatch: expected the suspended module program".to_string())
777                            }
778                        }
779                    }
780                }
781            },
782        };
783        #[cfg(test)]
784        if let Some(binding) = &self._program_binding {
785            binding.pause_after_wait_for_test().await;
786        }
787        if cause.is_none() && pending.gated {
788            let publication = pending.admission_binding.clone();
789            let live = self.live_program_binding.clone();
790            let signature = Arc::clone(pending.signature.as_ref().expect("gated signature"));
791            let pid = self.pid;
792            let gated = pending.gated;
793            self.privacy_exec
794                .as_mut()
795                .expect("pending exec")
796                .content_check = Some(tokio::task::spawn_blocking(move || {
797                signature.verify(pid).map_err(|error| {
798                    crate::config_gate::GateError::new("program_refused", error)
799                })?;
800                if !gated {
801                    return Ok(());
802                }
803                match (publication, live) {
804                    (Some(publication), _) => publication.verify(),
805                    (None, Some(live)) => live.check(),
806                    _ => Err(crate::config_gate::GateError::new(
807                        "config_delivery_deferred",
808                        "live program witness is unavailable",
809                    )),
810                }
811            }));
812            self.finish_privacy_content_check().await;
813        } else {
814            self.finish_privacy_exec(cause);
815        }
816    }
817
818    #[cfg(target_os = "macos")]
819    async fn finish_privacy_content_check(&mut self) {
820        let result = self
821            .privacy_exec
822            .as_mut()
823            .expect("pending exec")
824            .content_check
825            .as_mut()
826            .expect("pending content check")
827            .await;
828        let cause = match result {
829            Ok(Ok(())) => None,
830            Ok(Err(error)) => Some(format!("{}: {}", error.code, error.detail)),
831            Err(error) => Some(format!(
832                "config_delivery_deferred: content check stopped: {error}"
833            )),
834        };
835        self.finish_privacy_exec(cause);
836    }
837
838    #[cfg(target_os = "macos")]
839    fn finish_privacy_exec(&mut self, mut cause: Option<String>) {
840        let mut pending = self.privacy_exec.take().expect("pending exec");
841        if cause.is_none() && pending.gated {
842            let admission = (|| -> io::Result<()> {
843                let image = observe_spawned_image(self.pid)
844                    .filter(|image| image.executable == pending.expected)
845                    .ok_or_else(|| io::Error::other("suspended executable mismatch"))?;
846                if let Some(guard) = &self.roster_guard {
847                    guard.confirm_executable(image)?;
848                }
849                pending
850                    .signature
851                    .as_ref()
852                    .expect("gated signature")
853                    .resume(self.pid)?;
854                let _ = self.report_ready.set(());
855                info!(module_id = %pending.module_id, pid = self.pid,
856                    "verified suspended module resumed with own privacy identity");
857                Ok(())
858            })();
859            if let Err(error) = admission {
860                cause = Some(format!("program_refused: {error}"));
861            }
862        }
863        if cause.is_some() {
864            if let Some(spawn) = &self.registration_spawn {
865                spawn.refuse();
866            }
867            if let Some(binding) = &mut pending.admission_binding {
868                binding.reject();
869            }
870        } else if let Some(spawn) = &self.registration_spawn {
871            spawn.ready();
872        }
873        // Exec confirmation describes launch success, not roster retirement.
874        // Keep even a reaped child recorded until its terminal history exists.
875        self.exec.0.finish(if cause.is_some() {
876            crate::activity::ExecState::Refused
877        } else {
878            crate::activity::ExecState::CommittedExec
879        });
880        if let Some(cause) = cause {
881            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
882            self.spawn_failure = Some(
883                if pending.gated
884                    && (cause.contains("executable mismatch")
885                        || cause.starts_with("program_refused:"))
886                {
887                    format!("program_refused: {cause}")
888                } else if pending.gated {
889                    format!("config_delivery_deferred: {cause}")
890                } else {
891                    cause
892                },
893            );
894            // No resume signal is sent on failure; kill the entire fresh group
895            // even if a faulty trampoline did not honor the suspended protocol.
896            let _ = self.start_admission_kill();
897        }
898    }
899
900    /// Releases the leader's roster entry after recording and finalizing its
901    /// exit. A carried container retains its separate roster entry and, for a
902    /// Unix process group, its unreaped first leader that reserves the group ID.
903    fn release_roster(&mut self) {
904        self.roster_guard = None;
905    }
906
907    /// Kill the child and, where containment is available, its process tree.
908    ///
909    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
910    /// helper process leaked the helper — the Synapse embedding module's CUDA
911    /// worker holds the GPU allocation, so the leak cost VRAM until the next
912    /// restart of something else. Terminating the job reaches grandchildren that
913    /// a tree walk cannot, including one whose parent has already exited and
914    /// been reparented away.
915    ///
916    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
917    /// direct-child kill still decides the outcome, so containment can never
918    /// change whether a module is reported as stopped.
919    fn start_kill(&mut self) -> io::Result<()> {
920        self.retire_registration();
921        if let Some(container) = self.child.container() {
922            container.hard_kill();
923            return self.kill_leader();
924        }
925        #[cfg(windows)]
926        if let Some(job) = &self.job {
927            if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
928                debug!(
929                    error = %error,
930                    "job termination failed; the direct-child kill still owns the outcome"
931                );
932            }
933        }
934        #[cfg(target_os = "linux")]
935        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
936        self.child.start_kill()
937    }
938
939    fn kill_leader(&mut self) -> io::Result<()> {
940        self.retire_registration();
941        self.child.start_kill()
942    }
943
944    fn start_admission_kill(&mut self) -> io::Result<()> {
945        self.retire_registration();
946        #[cfg(windows)]
947        let result = {
948            // Preserve the leader's distinctive status before ending the whole
949            // container/job; descendants must not wait for exit finalization.
950            let result = self
951                .spawned_file_identity
952                .as_ref()
953                .and_then(|identity| identity.windows_image.as_ref())
954                .ok_or_else(|| io::Error::other("admission child handle unavailable"))
955                .and_then(|image| {
956                    image.process().terminate_with_exit_code(
957                        admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE as u32,
958                    )
959                });
960            if let Some(container) = self.child.container() {
961                container.hard_kill();
962            } else if let Some(job) = &self.job {
963                if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
964                    debug!(%error, "job termination failed after admission leader termination");
965                }
966            }
967            result
968        };
969        #[cfg(not(windows))]
970        let result = self.start_kill().map(|()| true);
971        #[cfg(unix)]
972        let result = {
973            // A leader can exit before its helpers. Still end the fresh group
974            // even when the direct-child kill reports that the leader is gone.
975            let group_sent = self.child.container().is_none()
976                && rustix::process::Pid::from_raw(self.pid as i32).is_some_and(|group| {
977                    rustix::process::kill_process_group(group, rustix::process::Signal::KILL)
978                        .is_ok()
979                });
980            if group_sent {
981                Ok(true)
982            } else {
983                result
984            }
985        };
986        self.admission_kill_sent |= result?;
987        Ok(())
988    }
989
990    async fn drain_stderr(&mut self, module_id: &str) {
991        if let Some(pump) = self.stdout_pump.take() {
992            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
993        }
994
995        let Some(pump) = self.stderr_pump.take() else {
996            return;
997        };
998        settle_stderr_pump(
999            module_id,
1000            &self.stderr_ring,
1001            pump,
1002            STDERR_PUMP_DRAIN_TIMEOUT,
1003        )
1004        .await;
1005    }
1006}
1007
1008/// Bound the restart's wait, not the lifetime of the exited child's stdout
1009/// reader. Slow pipe reads or capture writes must not turn the bound into lost
1010/// output; dropping the handle lets the reader finish at EOF, like stderr.
1011async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
1012    match timeout(bound, &mut pump).await {
1013        Ok(Ok(())) => {}
1014        Ok(Err(error)) => {
1015            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
1016        }
1017        Err(_) => {
1018            drop(pump);
1019            warn!(
1020                module_id,
1021                waited = ?bound,
1022                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
1023            );
1024        }
1025    }
1026}
1027
1028/// The reader task for one process's stderr, with the ring generation its
1029/// lines are attributed to.
1030struct StderrPump {
1031    task: JoinHandle<()>,
1032    generation: u64,
1033}
1034
1035/// Retire an exited process's stderr reader and wait up to `bound` for it to
1036/// reach EOF. A reader still running at the bound is detached, not stopped: it
1037/// keeps filling the exited process's section of the ring until its pipe
1038/// closes, and the tail reads `Incomplete` until then. See
1039/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
1040async fn settle_stderr_pump(
1041    module_id: &str,
1042    ring: &Arc<Mutex<StderrRing>>,
1043    pump: StderrPump,
1044    bound: Duration,
1045) {
1046    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
1047    let StderrPump {
1048        mut task,
1049        generation,
1050    } = pump;
1051    lock().retire_pump(generation);
1052    match timeout(bound, &mut task).await {
1053        Ok(Ok(())) => {}
1054        Ok(Err(err)) => {
1055            let mut ring = lock();
1056            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
1057            ring.finish_pump(generation);
1058            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
1059        }
1060        Err(_) => {
1061            // Dropping the handle detaches the task; it ends at EOF on its pipe.
1062            drop(task);
1063            lock().mark_pump_late(
1064                generation,
1065                format!(
1066                    "stderr of the exited process had not reached EOF {bound:?} after it was \
1067                     retired (a descendant may still hold the pipe open); lines it still \
1068                     writes are kept in that process's section"
1069                ),
1070            );
1071            warn!(
1072                module_id,
1073                waited = ?bound,
1074                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
1075            );
1076        }
1077    }
1078}
1079
1080fn registration_release_events() -> &'static watch::Sender<u64> {
1081    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
1082    EVENTS.get_or_init(|| {
1083        let (sender, _receiver) = watch::channel(0);
1084        sender
1085    })
1086}
1087
1088pub(crate) fn notify_registration_release() {
1089    let events = registration_release_events();
1090    let next_generation = (*events.borrow()).wrapping_add(1);
1091    events.send_replace(next_generation);
1092}
1093
1094/// The installed companion that evaluates a script module.
1095#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1096#[non_exhaustive]
1097pub enum ScriptRuntime {
1098    Bun,
1099    Python,
1100}
1101
1102impl ScriptRuntime {
1103    pub const fn as_str(self) -> &'static str {
1104        match self {
1105            Self::Bun => "bun",
1106            Self::Python => "python",
1107        }
1108    }
1109}
1110
1111/// Script content and the runtime approved to evaluate it.
1112#[derive(Debug, Clone, PartialEq, Eq)]
1113#[non_exhaustive]
1114pub struct ScriptEntry {
1115    pub runtime: ScriptRuntime,
1116    pub path: PathBuf,
1117}
1118
1119impl ScriptEntry {
1120    pub fn new(runtime: ScriptRuntime, path: PathBuf) -> Self {
1121        Self { runtime, path }
1122    }
1123}
1124
1125/// How to launch one singleton module process.
1126#[derive(Debug, Clone, PartialEq, Eq)]
1127pub struct ModuleSpec {
1128    pub module_id: String,
1129    pub program: PathBuf,
1130    /// Bundled script content. The program is always the installed runtime runner.
1131    pub script: Option<ScriptEntry>,
1132    pub args: Vec<String>,
1133    pub env: Vec<(String, String)>,
1134    /// When true this is a reserved module: each spawn gets a fresh one-time launch
1135    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
1136    /// process can register this module_id (a security-boundary module like the
1137    /// credential vault must not be impersonable while it is down/restarting).
1138    pub reserved: bool,
1139    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
1140    /// Prefixes come from daemon config and must end in `:` before they reach the
1141    /// supervisor; the owner module's current spawn nonce authorizes claims under
1142    /// each prefix.
1143    pub reserved_prefixes: Vec<String>,
1144    /// The wire protocol this module speaks, as DECLARED in daemon config.
1145    ///
1146    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
1147    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
1148    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
1149    /// and NO launch nonce, and a clean exit the daemon did not request is
1150    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
1151    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
1152    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
1153    /// because a process ignores an environment variable it does not read.
1154    ///
1155    /// The argument is the part that cannot be "harmless to a process that
1156    /// ignores it": a stock binary exits on an unknown flag before it listens
1157    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
1158    /// first conformance run against this mode found it. The nonce is withheld
1159    /// because a process that will never present it gains nothing from holding
1160    /// it, and a secret in the environment of a process that does not need it is
1161    /// a leak surface for no benefit.
1162    pub protocol: ModuleProtocol,
1163    /// Whether two processes of this module may run at once, which is what a
1164    /// blue/green swap does for the length of its overlap. Declared in daemon
1165    /// config because the daemon must be able to answer it while the module is
1166    /// down, and so a module cannot talk itself into it after registering.
1167    pub overlap: ModuleOverlap,
1168    /// Keep module-started helpers in the same container across replacements.
1169    /// This is a daemon-config policy, fixed for the lifetime of a container;
1170    /// stopping the module or losing its supervisor still destroys the container.
1171    pub carry_containment: bool,
1172}
1173
1174/// Only the configured launch inputs enter this value; spawned connection paths,
1175/// nonces and inherited environment never do. Its bytes remain daemon-private.
1176#[derive(Clone, PartialEq, Eq)]
1177pub(crate) struct LaunchSpecDigest([u8; 32]);
1178
1179impl fmt::Debug for LaunchSpecDigest {
1180    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1181        f.write_str("LaunchSpecDigest([redacted])")
1182    }
1183}
1184
1185impl LaunchSpecDigest {
1186    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
1187        fn field(hash: &mut Sha256, bytes: &[u8]) {
1188            hash.update((bytes.len() as u64).to_le_bytes());
1189            hash.update(bytes);
1190        }
1191        let mut hash = Sha256::new();
1192        if let Some(script) = &spec.script {
1193            field(
1194                &mut hash,
1195                match script.runtime {
1196                    ScriptRuntime::Bun => b"bun-script",
1197                    ScriptRuntime::Python => b"python-script",
1198                },
1199            );
1200            field(&mut hash, script.path.as_os_str().as_encoded_bytes());
1201        }
1202        if let Some(program) = spec.program.to_str() {
1203            field(&mut hash, b"utf8-program");
1204            field(&mut hash, program.as_bytes());
1205        } else {
1206            // The daemon config is UTF-8, but in-process callers may provide
1207            // native paths: lossy conversion would collapse distinct programs.
1208            #[cfg(unix)]
1209            {
1210                use std::os::unix::ffi::OsStrExt;
1211                field(&mut hash, b"unix-program");
1212                field(&mut hash, spec.program.as_os_str().as_bytes());
1213            }
1214            #[cfg(windows)]
1215            {
1216                use std::os::windows::ffi::OsStrExt;
1217                field(&mut hash, b"windows-program");
1218                let bytes = spec
1219                    .program
1220                    .as_os_str()
1221                    .encode_wide()
1222                    .flat_map(u16::to_le_bytes)
1223                    .collect::<Vec<_>>();
1224                field(&mut hash, &bytes);
1225            }
1226            #[cfg(not(any(unix, windows)))]
1227            field(&mut hash, spec.program.to_string_lossy().as_bytes());
1228        }
1229        hash.update((spec.args.len() as u64).to_le_bytes());
1230        for arg in &spec.args {
1231            field(&mut hash, arg.as_bytes());
1232        }
1233        let mut env = spec.env.iter().collect::<Vec<_>>();
1234        env.sort_unstable();
1235        hash.update((env.len() as u64).to_le_bytes());
1236        for (key, value) in env {
1237            field(&mut hash, key.as_bytes());
1238            field(&mut hash, value.as_bytes());
1239        }
1240        field(
1241            &mut hash,
1242            match spec.protocol {
1243                ModuleProtocol::Subc => b"subc",
1244                ModuleProtocol::None => b"none",
1245            },
1246        );
1247        Self(hash.finalize().into())
1248    }
1249}
1250
1251/// Whether a module tolerates a second process of itself running alongside.
1252///
1253/// Most modules are single-writer on their store (a WAL, a capture log, a
1254/// resident index behind a writer barrier), and two processes on one store
1255/// corrupt it. So a swap, which overlaps the old and new process by design,
1256/// is refused unless the module's config opts in with `overlap: "safe"`.
1257#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1258pub enum ModuleOverlap {
1259    /// Never run two processes of this module at once. The default.
1260    #[default]
1261    Exclusive,
1262    /// The module has said a second process of itself is harmless for the
1263    /// length of a swap.
1264    ///
1265    /// Declare it only if a second instance can run for a few seconds without
1266    /// touching ANY single-writer store: every database, WAL, index, projector
1267    /// and scheduled job the module owns. A lease on part of that state is not
1268    /// enough. broca's session lease guards WAL appends while its run index, its
1269    /// store projector and its archive fold timer (which unlinks live WAL files)
1270    /// stay single-writer, so broca is exclusive despite holding a lease. The
1271    /// refusal only fires after this has been decided, so the decision is the
1272    /// check.
1273    Safe,
1274}
1275
1276impl ModuleOverlap {
1277    pub fn as_str(self) -> &'static str {
1278        match self {
1279            Self::Exclusive => "exclusive",
1280            Self::Safe => "safe",
1281        }
1282    }
1283}
1284
1285/// Environment variable telling a spawned module which case it was started
1286/// for, before it sends HELLO. Only a swap candidate carries it, as
1287/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
1288///
1289/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
1290/// longer because nobody waits on it, while a plain restart must flip ready
1291/// quickly because callers see `module_warming` until it does. Absence means
1292/// plain restart, the safe reading. The daemon trusts nothing about it; the
1293/// candidate is proven by its launch nonce at HELLO.
1294pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
1295/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
1296pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
1297/// How long a swap waits for its candidate to register and declare itself
1298/// ready when the operator does not say. A module warming as a swap candidate
1299/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
1300/// daemon allows that plus time to start the process and send HELLO.
1301pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
1302
1303/// Bounded restart policy for crash exits.
1304///
1305/// `max_restarts` is the number of replacement processes allowed after the
1306/// initial spawn WITHIN `window`. After that many crash restarts inside one
1307/// window the module enters [`ModuleState::Failed`] and the supervisor stops
1308/// the crash loop.
1309///
1310/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
1311/// and that only survived because crashes were rare: a module that crashed
1312/// three times across a week was disabled forever by crashes that had nothing
1313/// to do with each other. That stopped being survivable once modules began
1314/// exiting non-zero whenever the daemon's connection to them drops, because
1315/// then every daemon-side connection drop spends a unit of the same budget and
1316/// one flappy hour permanently stops a healthy module. Restarts older than
1317/// `window` release their slot, so a module that crashed twice yesterday has a
1318/// full budget today, while a genuine crash loop -- which is fast by
1319/// definition -- still reaches the cap and stops.
1320#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1321pub struct RestartPolicy {
1322    pub max_restarts: u32,
1323    /// Base delay before a crash replacement. The actual delay escalates with
1324    /// the number of recent crash replacements and is capped by `max_backoff`.
1325    pub backoff: Duration,
1326    /// Maximum delay before a crash replacement.
1327    pub max_backoff: Duration,
1328    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
1329    /// budget effectively infinite (nothing is ever in-window), which is why
1330    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
1331    pub window: Duration,
1332}
1333
1334impl RestartPolicy {
1335    /// A policy with the default crash window. Callers that care about the
1336    /// window say so with [`Self::with_window`]; the ones that do not are
1337    /// asking for the standard rate limit, not for no limit.
1338    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
1339        Self {
1340            max_restarts,
1341            backoff,
1342            max_backoff: DEFAULT_MAX_BACKOFF,
1343            window: DEFAULT_RESTART_WINDOW,
1344        }
1345    }
1346
1347    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
1348        self.max_backoff = max_backoff;
1349        self
1350    }
1351
1352    pub fn with_window(mut self, window: Duration) -> Self {
1353        self.window = window;
1354        self
1355    }
1356
1357    /// Calculate the capped exponential delay for the next crash replacement.
1358    /// `restart_in_window` is zero for the first replacement after an operator
1359    /// action (restart, reload, re-enable) cleared the crash ring, or after all
1360    /// older crash replacements have aged out of the window.
1361    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
1362        if self.backoff.is_zero() || self.max_backoff.is_zero() {
1363            return Duration::ZERO;
1364        }
1365
1366        let mut delay = self.backoff;
1367        for _ in 0..restart_in_window {
1368            if delay >= self.max_backoff {
1369                return self.max_backoff;
1370            }
1371            delay = delay
1372                .checked_mul(10)
1373                .unwrap_or(self.max_backoff)
1374                .min(self.max_backoff);
1375        }
1376        delay.min(self.max_backoff)
1377    }
1378
1379    /// The one sentence that explains a budget-exhausted stop, used for both the
1380    /// log line and the terminal record so the two cannot drift. It names the
1381    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
1382    /// exactly what this budget is not.
1383    fn budget_exhausted_detail(&self) -> String {
1384        format!(
1385            "crash budget exhausted: max_restarts={} within window_secs={}",
1386            self.max_restarts,
1387            self.window.as_secs()
1388        )
1389    }
1390}
1391
1392impl Default for RestartPolicy {
1393    fn default() -> Self {
1394        Self {
1395            max_restarts: DEFAULT_MAX_RESTARTS,
1396            backoff: DEFAULT_BACKOFF,
1397            max_backoff: DEFAULT_MAX_BACKOFF,
1398            window: DEFAULT_RESTART_WINDOW,
1399        }
1400    }
1401}
1402
1403#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1404struct CrashRestartSchedule {
1405    restart_in_window: u32,
1406    delay: Duration,
1407}
1408
1409/// Whether the daemon itself will bring this module back after the exit being
1410/// handled: it is enabled AND its in-window crash restarts are below the cap.
1411///
1412/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
1413/// the window are dropped here rather than by a timer, so the count is right
1414/// the moment somebody asks and no bookkeeping runs for idle modules.
1415fn daemon_will_restart(
1416    state: &mut SupervisorSnapshot,
1417    policy: &RestartPolicy,
1418    now: Instant,
1419) -> bool {
1420    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1421}
1422
1423const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1424const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1425const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1426const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1427
1428#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1429pub enum HealthAction {
1430    Report,
1431    Restart,
1432    Alert,
1433}
1434
1435impl fmt::Display for HealthAction {
1436    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1437        f.write_str(match self {
1438            Self::Report => "report",
1439            Self::Restart => "restart",
1440            Self::Alert => "alert",
1441        })
1442    }
1443}
1444
1445#[derive(Debug, Clone, PartialEq, Eq)]
1446pub struct HealthConfig {
1447    /// Optional loopback HTTP endpoint for a managed non-wire process.
1448    /// Changing it applies live on rescan; the process protocol changes only
1449    /// at its next spawn.
1450    pub http: Option<String>,
1451    pub cadence: Duration,
1452    pub deadline: Duration,
1453    pub failure_threshold: u32,
1454    pub on_degraded: HealthAction,
1455    pub on_failing: HealthAction,
1456    pub critical: bool,
1457}
1458
1459impl Default for HealthConfig {
1460    fn default() -> Self {
1461        Self {
1462            http: None,
1463            cadence: DEFAULT_HEALTH_CADENCE,
1464            deadline: DEFAULT_HEALTH_DEADLINE,
1465            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1466            on_degraded: HealthAction::Report,
1467            on_failing: HealthAction::Report,
1468            critical: false,
1469        }
1470    }
1471}
1472
1473/// The supervisor's view of one module's health, relayed to clients over
1474/// channel-0 and rendered by `ck health`.
1475///
1476/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1477/// stated here rather than only at the wire type a consumer reads. A reader can
1478/// look up what `None` means; only a writer can silently change it, and the
1479/// writer has no reason to go looking at a downstream contract before editing.
1480///
1481/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1482/// back to `None` on re-registration precisely so a respawned module does not
1483/// carry its predecessor's timestamp — so an old value and an absent one call for
1484/// opposite readings, and anything that defaulted this to a number would make a
1485/// never-probed module indistinguishable from one probed at the epoch.
1486///
1487/// `detail` and `metrics` are `None` when the module published none on this
1488/// probe, which does not mean it reported nothing wrong — it is also the shape
1489/// when the probe never reached it. `last_probe_ms` is what separates those.
1490#[derive(Debug, Clone, PartialEq)]
1491pub struct ModuleHealthStatus {
1492    pub status: SupervisorHealthStatus,
1493    pub last_probe_ms: Option<u64>,
1494    pub detail: Option<String>,
1495    pub metrics: Option<Value>,
1496    pub consecutive_failures: u32,
1497    /// Number of replies received after a recurring health probe's deadline.
1498    /// Unlike a timeout, every increment proves the module was alive.
1499    pub late_answer_count: u64,
1500    /// End-to-end latency of the newest late reply, measured from probe start.
1501    pub last_late_answer_latency_ms: Option<u64>,
1502    pub last_action: Option<String>,
1503    /// Set together with `last_action`; the pair moves as one, and both being
1504    /// absent means no escalation has ever been taken rather than that the last
1505    /// one succeeded.
1506    pub last_action_ms: Option<u64>,
1507}
1508
1509impl Default for ModuleHealthStatus {
1510    fn default() -> Self {
1511        Self {
1512            status: SupervisorHealthStatus::Unknown,
1513            last_probe_ms: None,
1514            detail: None,
1515            metrics: None,
1516            consecutive_failures: 0,
1517            late_answer_count: 0,
1518            last_late_answer_latency_ms: None,
1519            last_action: None,
1520            last_action_ms: None,
1521        }
1522    }
1523}
1524
1525/// Typed lifecycle state for a supervised module.
1526#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1527pub enum ModuleState {
1528    Starting,
1529    Running,
1530    Unresponsive,
1531    Restarting,
1532    Draining,
1533    Stopped,
1534    Failed,
1535    Disabled,
1536}
1537
1538impl fmt::Display for ModuleState {
1539    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1540        f.write_str(match self {
1541            Self::Starting => "starting",
1542            Self::Running => "running",
1543            Self::Unresponsive => "unresponsive",
1544            Self::Restarting => "restarting",
1545            Self::Draining => "draining",
1546            Self::Stopped => "stopped",
1547            Self::Failed => "failed",
1548            Self::Disabled => "disabled",
1549        })
1550    }
1551}
1552
1553/// Supervisor classification of a child-process exit.
1554#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1555pub enum ExitKind {
1556    Clean,
1557    Crash,
1558    DeliberateSeverance,
1559}
1560
1561impl From<ExitKind> for TerminalExitKind {
1562    fn from(kind: ExitKind) -> Self {
1563        match kind {
1564            ExitKind::Clean => Self::Clean,
1565            ExitKind::Crash => Self::Crash,
1566            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1567        }
1568    }
1569}
1570
1571/// Exact process identity retained when a supervised module registers its
1572/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1573#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1574pub(crate) struct ProcessIdentity {
1575    pub(crate) pid: u32,
1576    pub(crate) start_time: u64,
1577}
1578
1579/// Last observed child exit, if any.
1580#[derive(Debug, Clone, PartialEq, Eq)]
1581pub struct ExitReport {
1582    pub kind: ExitKind,
1583    pub code: Option<i32>,
1584    pub signal: Option<i32>,
1585    pub at_ms: u64,
1586}
1587
1588/// Point-in-time module status answerable by subc without forwarding to the
1589/// module process.
1590#[derive(Debug, Clone, PartialEq)]
1591pub struct ModuleStatus {
1592    pub(crate) admission_refused: Option<String>,
1593    pub(crate) legacy_registration_spawn: bool,
1594    pub(crate) registration_declaration: Option<String>,
1595    pub module_id: String,
1596    /// Retained configuration-exit infrastructure failure. Cleared only after a
1597    /// successful launch; an ordinary stop has no infrastructure stop reason.
1598    pub stop_reason: Option<ModuleStopReason>,
1599    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1600    pub waiting_for_previous_pid: Option<u32>,
1601    /// Why the previous instance still prevents a new start.
1602    pub waiting_for_previous_detail: Option<String>,
1603    /// The daemon's Windows per-spawn record: `pipe` after the process it
1604    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1605    /// List/status reports use this record, not the source the module sends
1606    /// in its registration message's manifest.provenance.launch_nonce_source.
1607    pub launch_nonce_source: Option<String>,
1608    pub state: ModuleState,
1609    pub enabled: bool,
1610    pub process_alive: bool,
1611    pub registration_active: bool,
1612    /// The module's declared wire protocol, carried beside `live` because it is
1613    /// what makes `live` readable: the two fields answer one question together.
1614    /// While a process is alive this is its launch declaration, not a later
1615    /// pending-reload edit. When down it is the configured next launch protocol.
1616    pub protocol: ModuleProtocol,
1617    /// Whether the module is serving, under the strongest definition the daemon
1618    /// can assert for its protocol.
1619    ///
1620    /// A subc module must also be REGISTERED: its process being alive says
1621    /// nothing about whether it can take a request. A `protocol: "none"` module
1622    /// never registers, so that term is dropped and this falls back to "enabled,
1623    /// running, and the process the daemon launched is alive" -- which is all
1624    /// the daemon observes about a process that speaks no subc wire. It stays a
1625    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1626    /// rather than printing it bare.
1627    pub live: bool,
1628    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1629    /// restarts have already released their slot, so this count can go down
1630    /// without anybody touching the module.
1631    pub restart_count: u32,
1632    /// Replacement processes spawned over this module's entire supervisor lifetime;
1633    /// unlike `restart_count`, this value is never reset by an operator action
1634    /// and never falls out of a window.
1635    pub lifetime_restarts: u32,
1636    pub spawn_generation: u64,
1637    /// Retained daemon-local outcome; the legacy control response omits it.
1638    pub swap_outcome: Option<SwapOutcome>,
1639    /// The budget `restart_count` is spent against. Carried alongside the count
1640    /// because the count alone does not say how close the module is to being
1641    /// disabled, and reporting one without the other is what makes an
1642    /// about-to-be-retired module look ordinary.
1643    pub max_restarts: u32,
1644    /// The span `restart_count` is counted over. Carried with the pair above for
1645    /// the same reason they are carried together: "2 of 3" means one thing for a
1646    /// ten-minute window and something else entirely for a lifetime.
1647    pub restart_window: Duration,
1648    /// Effective drain and restart timing policy used by this running module.
1649    /// These values are carried together with the restart budget so status
1650    /// readers can compare configured intent with what the supervisor applied.
1651    pub drain_timeout: Duration,
1652    pub restart_backoff: Duration,
1653    pub restart_max_backoff: Duration,
1654    /// The module's process. On macOS this stays absent while the `ck-subc`
1655    /// launch trampoline is still running in that pid, and appears once the
1656    /// exec acknowledgement confirms the module image has replaced it. Launch
1657    /// time and the supervisor's own hold on the process are unaffected.
1658    pub pid: Option<u32>,
1659    pub spawned_at_ms: Option<u64>,
1660    pub spawned_from: Option<PathBuf>,
1661    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1662    pub process_start_time: Option<u64>,
1663    pub last_exit: Option<ExitReport>,
1664    pub health: ModuleHealthStatus,
1665}
1666
1667#[cfg(test)]
1668#[derive(Debug, Clone, Copy, PartialEq)]
1669struct ActorSelectCheckpoint {
1670    generation: u64,
1671    turn: u64,
1672    registered_connection: Option<ConnectionId>,
1673    next_probe_at: Option<Instant>,
1674    wake_after: Option<Duration>,
1675}
1676
1677#[derive(Debug, Clone, PartialEq)]
1678struct SupervisorSnapshot {
1679    admission_refused: Option<String>,
1680    /// Distinguishes an admission-parked actor from an unrelated terminal exit.
1681    config_hold_parked: bool,
1682    /// Preserve daemon attribution for a paused start so a failing release can
1683    /// schedule the next slow retry instead of treating the start as operator work.
1684    config_hold_daemon_initiated: bool,
1685    /// The approval that owns the parked attempt, not whichever approval is latest.
1686    config_hold_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1687    /// Retain an approved launch failure across competing reconciliation passes.
1688    approved_start_failure: Option<(crate::config_gate::runtime::ApprovalIdentity, String)>,
1689    legacy_registration_spawn: bool,
1690    registration_declaration: Option<String>,
1691    stop_reason: Option<ModuleStopReason>,
1692    config_exit_failed_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1693    subtree_target: crate::subtree_resources::Target,
1694    failed_retry: FailedRetry,
1695    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1696    #[cfg(windows)]
1697    pipe_consumed: Option<Arc<OnceLock<()>>>,
1698    /// Counts running-child loop turns, not executor polls of a parked wait.
1699    #[cfg(test)]
1700    actor_turns: u64,
1701    /// Running select after exec confirmation and with no unconsumed registry
1702    /// event. Recording a spawn or starting a loop turn is not this barrier.
1703    #[cfg(test)]
1704    actor_select: Option<ActorSelectCheckpoint>,
1705    state: ModuleState,
1706    enabled: bool,
1707    process_alive: bool,
1708    spawned_protocol: Option<ModuleProtocol>,
1709    spawn_failure: Option<String>,
1710    waiting_for_previous_pid: Option<u32>,
1711    waiting_for_previous_detail: Option<String>,
1712    /// When each crash restart was spent, oldest first. This IS the crash
1713    /// budget: its in-window length is the count an operator sees and the count
1714    /// the restart decision is made against, so there is no second counter that
1715    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1716    /// operator actions that used to zero the old lifetime counter.
1717    crash_restarts: VecDeque<Instant>,
1718    lifetime_restarts: u32,
1719    /// Successful child spawns in this daemon incarnation.
1720    ///
1721    /// `lifetime_restarts` was considered and rejected: it starts at zero
1722    /// (line 640), successful initial/operator spawns in `set_running` do not
1723    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1724    /// increments before a successful replacement exists (lines 604, 3846,
1725    /// and 3921), so a failed spawn can consume it. This counter moves only
1726    /// when a live PID is accepted below.
1727    spawn_generation: u64,
1728    swap_outcome: Option<SwapOutcome>,
1729    pid: Option<u32>,
1730    #[cfg(target_os = "macos")]
1731    report_ready: Option<Arc<OnceLock<()>>>,
1732    /// Last reaped child, retained after current process facts are cleared.
1733    reaped_pid: Option<u32>,
1734    /// Whether a replacement has already been scheduled for the current process.
1735    respawn_pending: bool,
1736    /// A second restart is waiting for the replacement already scheduled.
1737    coalesced_restart_pending: bool,
1738    spawned_at_ms: Option<u64>,
1739    spawned_from: Option<PathBuf>,
1740    launch_spec_digest: Option<LaunchSpecDigest>,
1741    spawned_file_identity: Option<SpawnedFileIdentity>,
1742    process_start_time: Option<u64>,
1743    deliberate_severance: Option<ProcessIdentity>,
1744    last_exit: Option<ExitReport>,
1745    /// Diagnostic attached to the next drain's terminal record, if any.
1746    drain_disposition_detail: Option<String>,
1747    health: ModuleHealthStatus,
1748    last_probe_evidence: Option<HealthProbeEvidence>,
1749    /// Whether the current process was started as a swap candidate and so
1750    /// lives in the module's alternate cgroup. The next swap's candidate takes
1751    /// the other one, so the two processes of a swap never share a cgroup. A
1752    /// plain spawn always uses the primary cgroup.
1753    in_alternate_slot: bool,
1754    /// Whether the current `Draining` state ends in a replacement process
1755    /// (restart, reload, health restart) rather than a stop. Only meaningful
1756    /// while `state` is `Draining`; every entry into that state rewrites it.
1757    /// It is what lets route.open answer the retryable `module_reloading` to a
1758    /// consumer that reaches a still-registered process mid-restart, instead of
1759    /// the `supervisor_not_live` a stop or disable deserves.
1760    draining_to_replace: bool,
1761    /// Whether a configuration update has been applied since the current
1762    /// process was spawned, so that process runs an older spec than the one
1763    /// the supervisor now holds. A queued restart is only coalesced into a
1764    /// fresher process when this is false: a restart requested to pick up a
1765    /// new configuration must not be satisfied by a process that predates it.
1766    configuration_updated_since_spawn: bool,
1767}
1768
1769impl SupervisorSnapshot {
1770    /// The pid that status, provenance and resource readings may report. While
1771    /// the launch trampoline still runs in the pid, reading its executable or
1772    /// resource use would describe `ck-subc`, not the module, so none is
1773    /// reported until the exec acknowledgement confirms the module image.
1774    fn reported_pid(&self) -> Option<u32> {
1775        #[cfg(target_os = "macos")]
1776        if self
1777            .report_ready
1778            .as_ref()
1779            .is_some_and(|ready| ready.get().is_none())
1780        {
1781            return None;
1782        }
1783        self.pid
1784    }
1785
1786    fn starting() -> Self {
1787        Self::new(ModuleState::Starting, true)
1788    }
1789
1790    fn disabled() -> Self {
1791        Self::new(ModuleState::Disabled, false)
1792    }
1793
1794    /// Crash restarts still inside `window`, having dropped the ones that are
1795    /// not. Pruning on read is what makes the budget a rate: an instant older
1796    /// than the window stops holding a slot the moment anybody counts.
1797    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1798        while let Some(oldest) = self.crash_restarts.front() {
1799            if now.duration_since(*oldest) > window {
1800                self.crash_restarts.pop_front();
1801            } else {
1802                break;
1803            }
1804        }
1805        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1806    }
1807
1808    /// Spend one unit of the crash budget and record the restart in the ledger.
1809    ///
1810    /// The ring is bounded by the cap because more than `max_restarts` in-window
1811    /// instants can never be reached (the caller refuses the restart first), so
1812    /// anything beyond that is an unbounded queue waiting to happen.
1813    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1814        self.crash_restarts.push_back(now);
1815        while self.crash_restarts.len() > policy.max_restarts as usize {
1816            self.crash_restarts.pop_front();
1817        }
1818        self.lifetime_restarts += 1;
1819    }
1820
1821    /// Reserve one crash-restart slot and calculate the delay before respawning.
1822    /// The count is captured before recording this restart, so the first retry
1823    /// uses the base delay and each later in-window retry escalates once.
1824    fn next_crash_restart(
1825        &mut self,
1826        policy: &RestartPolicy,
1827        now: Instant,
1828    ) -> Option<CrashRestartSchedule> {
1829        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1830        if restart_in_window >= policy.max_restarts {
1831            return None;
1832        }
1833        self.record_crash_restart(policy, now);
1834        Some(CrashRestartSchedule {
1835            restart_in_window,
1836            delay: policy.delay_for_restart(restart_in_window),
1837        })
1838    }
1839
1840    /// Give the module its full budget back, as an operator restart, reload, or
1841    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1842    /// ledger of what actually happened, and an operator action does not unmake
1843    /// the crashes.
1844    fn clear_crash_restarts(&mut self) {
1845        self.crash_restarts.clear();
1846    }
1847
1848    fn new(state: ModuleState, enabled: bool) -> Self {
1849        Self {
1850            admission_refused: None,
1851            config_hold_parked: false,
1852            config_hold_daemon_initiated: false,
1853            config_hold_approval: None,
1854            approved_start_failure: None,
1855            legacy_registration_spawn: false,
1856            registration_declaration: None,
1857            stop_reason: None,
1858            config_exit_failed_approval: None,
1859            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1860            failed_retry: FailedRetry::default(),
1861            exec_outcome: None,
1862            #[cfg(test)]
1863            actor_turns: 0,
1864            #[cfg(test)]
1865            actor_select: None,
1866            state,
1867            enabled,
1868            process_alive: false,
1869            #[cfg(windows)]
1870            pipe_consumed: None,
1871            spawned_protocol: None,
1872            spawn_failure: None,
1873            waiting_for_previous_pid: None,
1874            waiting_for_previous_detail: None,
1875            crash_restarts: VecDeque::new(),
1876            lifetime_restarts: 0,
1877            spawn_generation: 0,
1878            swap_outcome: None,
1879            pid: None,
1880            #[cfg(target_os = "macos")]
1881            report_ready: None,
1882            reaped_pid: None,
1883            respawn_pending: false,
1884            coalesced_restart_pending: false,
1885            spawned_at_ms: None,
1886            spawned_from: None,
1887            launch_spec_digest: None,
1888            spawned_file_identity: None,
1889            process_start_time: None,
1890            deliberate_severance: None,
1891            last_exit: None,
1892            drain_disposition_detail: None,
1893            health: ModuleHealthStatus::default(),
1894            last_probe_evidence: None,
1895            in_alternate_slot: false,
1896            draining_to_replace: false,
1897            configuration_updated_since_spawn: false,
1898        }
1899    }
1900}
1901
1902type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1903
1904type SpawnSubscriberKey = (ConnectionId, u64);
1905
1906#[derive(Debug)]
1907struct SpawnSubscriber {
1908    version: u8,
1909    frames: mpsc::Sender<Frame>,
1910    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1911    /// from which event. The full frame channel cannot carry that news, so it
1912    /// travels beside it; see `SpawnEventFeed::subscribe`.
1913    lagged: Option<oneshot::Sender<SpawnCursor>>,
1914}
1915
1916#[derive(Debug)]
1917struct SpawnEventState {
1918    daemon_incarnation: String,
1919    seq: u64,
1920    capacity: usize,
1921    live: HashMap<String, LiveSpawn>,
1922    generations: HashMap<String, u64>,
1923    events: VecDeque<SpawnEvent>,
1924    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1925    process_changes: HashMap<String, tokio::sync::watch::Sender<u64>>,
1926}
1927
1928impl Default for SpawnEventState {
1929    fn default() -> Self {
1930        Self {
1931            daemon_incarnation: "unconfigured".to_string(),
1932            seq: 0,
1933            capacity: SPAWN_EVENT_RING_CAPACITY,
1934            live: HashMap::new(),
1935            generations: HashMap::new(),
1936            events: VecDeque::new(),
1937            subscribers: HashMap::new(),
1938            process_changes: HashMap::new(),
1939        }
1940    }
1941}
1942
1943#[derive(Debug, Clone, Default)]
1944struct SpawnEventFeed(
1945    Arc<Mutex<SpawnEventState>>,
1946    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1947);
1948
1949#[derive(Debug, Clone, PartialEq, Eq)]
1950pub(crate) enum SpawnSubscribeRefusal {
1951    ForeignIncarnation { current: String },
1952    TooOld { oldest: SpawnCursor },
1953    Frame(String),
1954}
1955
1956impl SpawnEventFeed {
1957    fn configure_incarnation(&self, daemon_incarnation: String) {
1958        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1959        state.daemon_incarnation = daemon_incarnation;
1960        state.seq = 0;
1961        state.live.clear();
1962        state.generations.clear();
1963        state.events.clear();
1964        state.subscribers.clear();
1965        for changed in state.process_changes.values() {
1966            changed.send_replace(0);
1967        }
1968    }
1969
1970    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1971        SpawnCursor {
1972            daemon_incarnation: state.daemon_incarnation.clone(),
1973            seq: state.seq,
1974        }
1975    }
1976
1977    fn snapshot(&self) -> SpawnSnapshot {
1978        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1979        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1980        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1981        SpawnSnapshot {
1982            cursor: Self::cursor(&state),
1983            ring_bound: state.capacity as u64,
1984            live,
1985        }
1986    }
1987
1988    fn allocate_generation_locked(state: &mut SpawnEventState, module_id: &str) -> u64 {
1989        let generation = state
1990            .generations
1991            .get(module_id)
1992            .copied()
1993            .unwrap_or(0)
1994            .checked_add(1)
1995            .expect("spawn generation exhausted");
1996        state.generations.insert(module_id.to_string(), generation);
1997        generation
1998    }
1999
2000    /// Allocate the spawn generation of a launch that is about to start, with
2001    /// the daemon incarnation it belongs to. The number names the launch from
2002    /// this moment, before its process runs; a launch that never runs keeps
2003    /// its number, so the generations a subscriber sees may skip.
2004    fn allocate_generation(&self, module_id: &str) -> (String, u64) {
2005        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2006        let generation = Self::allocate_generation_locked(&mut state, module_id);
2007        (state.daemon_incarnation.clone(), generation)
2008    }
2009
2010    /// Publish a running process. `allocated` is the generation its launch
2011    /// was given before it started; a launch without one (a process with no
2012    /// registration channel) is numbered here.
2013    fn emit_spawned(
2014        &self,
2015        module_id: &str,
2016        allocated: Option<u64>,
2017        pid: u32,
2018        spawned_at_ms: u64,
2019    ) -> u64 {
2020        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2021        let generation = match allocated {
2022            Some(generation) => generation,
2023            None => Self::allocate_generation_locked(&mut state, module_id),
2024        };
2025        let live = LiveSpawn {
2026            module_id: module_id.to_string(),
2027            spawn_generation: generation,
2028            pid,
2029            spawned_at_ms,
2030        };
2031        state.live.insert(module_id.to_string(), live);
2032        Self::emit_locked(
2033            &mut state,
2034            SpawnEventKind::Spawned,
2035            module_id.to_string(),
2036            generation,
2037            pid,
2038            None,
2039            None,
2040        );
2041        generation
2042    }
2043
2044    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
2045        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2046        let Some(live) = state.live.remove(module_id) else {
2047            warn!(
2048                module_id,
2049                "terminal record had no live spawn event identity"
2050            );
2051            return;
2052        };
2053        Self::emit_locked(
2054            &mut state,
2055            SpawnEventKind::Exited,
2056            module_id.to_string(),
2057            live.spawn_generation,
2058            live.pid,
2059            exit_code,
2060            exit_signal,
2061        );
2062    }
2063
2064    /// Report the exit of a process that a swap has already replaced.
2065    ///
2066    /// `emit_exited` removes the module's live entry, which after a swap's
2067    /// cutover describes the promoted candidate, not the old process now
2068    /// exiting. This emits the old generation's exit and leaves the live entry
2069    /// alone unless it still names that generation.
2070    fn emit_superseded_exited(
2071        &self,
2072        module_id: &str,
2073        spawn_generation: u64,
2074        pid: u32,
2075        exit_code: Option<i32>,
2076        exit_signal: Option<i32>,
2077    ) {
2078        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2079        if state
2080            .live
2081            .get(module_id)
2082            .is_some_and(|live| live.spawn_generation == spawn_generation)
2083        {
2084            state.live.remove(module_id);
2085        }
2086        Self::emit_locked(
2087            &mut state,
2088            SpawnEventKind::Exited,
2089            module_id.to_string(),
2090            spawn_generation,
2091            pid,
2092            exit_code,
2093            exit_signal,
2094        );
2095    }
2096
2097    #[allow(clippy::too_many_arguments)]
2098    fn emit_locked(
2099        state: &mut SpawnEventState,
2100        kind: SpawnEventKind,
2101        module_id: String,
2102        spawn_generation: u64,
2103        pid: u32,
2104        exit_code: Option<i32>,
2105        exit_signal: Option<i32>,
2106    ) {
2107        state.seq = state
2108            .seq
2109            .checked_add(1)
2110            .expect("spawn event sequence exhausted");
2111        let event = SpawnEvent {
2112            cursor: Self::cursor(state),
2113            kind,
2114            module_id,
2115            spawn_generation,
2116            pid,
2117            exit_code,
2118            exit_signal,
2119        };
2120        state
2121            .process_changes
2122            .retain(|_, changed| changed.receiver_count() > 0);
2123        if let Some(changed) = state.process_changes.get(&event.module_id) {
2124            changed.send_replace(state.seq);
2125        }
2126        state.events.push_back(event.clone());
2127        while state.events.len() > state.capacity {
2128            state.events.pop_front();
2129        }
2130        let body = match serde_json::to_vec(&event) {
2131            Ok(body) => body,
2132            Err(error) => {
2133                error!(%error, "failed to serialize supervisor spawn event");
2134                return;
2135            }
2136        };
2137        state.subscribers.retain(|(connection_id, corr), subscriber| {
2138            let frame = Frame::build_with_version(
2139                subscriber.version,
2140                FrameType::StreamData,
2141                control_flags(),
2142                0,
2143                0,
2144                *corr,
2145                body.clone(),
2146            );
2147            match frame {
2148                Ok(frame) => {
2149                    if subscriber.frames.try_send(frame).is_ok() {
2150                        true
2151                    } else {
2152                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
2153                        if let Some(lagged) = subscriber.lagged.take() {
2154                            let _ = lagged.send(event.cursor.clone());
2155                        }
2156                        false
2157                    }
2158                }
2159                Err(error) => {
2160                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
2161                    false
2162                }
2163            }
2164        });
2165    }
2166
2167    fn subscribe(
2168        &self,
2169        connection_id: ConnectionId,
2170        corr: u64,
2171        version: u8,
2172        since: Option<SpawnCursor>,
2173        sink: FrameSink,
2174    ) -> Result<(), SpawnSubscribeRefusal> {
2175        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
2176        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
2177        {
2178            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2179            let replay = if let Some(since) = since {
2180                if since.daemon_incarnation != state.daemon_incarnation {
2181                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
2182                        current: state.daemon_incarnation.clone(),
2183                    });
2184                }
2185                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
2186                    if since.seq < oldest.seq.saturating_sub(1) {
2187                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
2188                    }
2189                }
2190                state
2191                    .events
2192                    .iter()
2193                    .filter(|event| event.cursor.seq > since.seq)
2194                    .cloned()
2195                    .collect::<Vec<_>>()
2196            } else {
2197                Vec::new()
2198            };
2199            for event in replay {
2200                let body = serde_json::to_vec(&event)
2201                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2202                let frame = Frame::build_with_version(
2203                    version,
2204                    FrameType::StreamData,
2205                    control_flags(),
2206                    0,
2207                    0,
2208                    corr,
2209                    body,
2210                )
2211                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2212                frames
2213                    .try_send(frame)
2214                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2215            }
2216            state.subscribers.insert(
2217                (connection_id, corr),
2218                SpawnSubscriber {
2219                    version,
2220                    frames,
2221                    lagged: Some(lagged),
2222                },
2223            );
2224        }
2225        // The lagged terminal is sent here, by the forwarder, rather than by
2226        // the emitter: at the moment of the drop the subscriber's own channel
2227        // is full, and writing to the connection sink directly from the emitter
2228        // would put the Error AHEAD of the events still queued in that channel
2229        // (and the emitter holds the feed lock, so it cannot await the sink).
2230        // Dropping the subscriber drops the only sender, so `recv` drains every
2231        // queued event and then returns `None`; only then is the Error sent, so
2232        // the client sees each event it can keep, then the reason it was cut.
2233        // Cancel and connection removal drop the oneshot unsent, so they end
2234        // the stream with no Error.
2235        tokio::spawn(async move {
2236            while let Some(frame) = receiver.recv().await {
2237                if sink.send(frame).await.is_err() {
2238                    return;
2239                }
2240            }
2241            let Ok(first_undelivered) = lagged_rx.try_recv() else {
2242                return;
2243            };
2244            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
2245                Ok(frame) => {
2246                    let _ = sink.send(frame).await;
2247                }
2248                Err(error) => {
2249                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
2250                }
2251            }
2252        });
2253        Ok(())
2254    }
2255
2256    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
2257        let Some(subscriber) = self
2258            .0
2259            .lock()
2260            .unwrap_or_else(|p| p.into_inner())
2261            .subscribers
2262            .remove(&(connection_id, corr))
2263        else {
2264            return false;
2265        };
2266        if let Ok(frame) = Frame::build_with_version(
2267            subscriber.version,
2268            FrameType::StreamEnd,
2269            control_flags(),
2270            0,
2271            0,
2272            corr,
2273            Vec::new(),
2274        ) {
2275            tokio::spawn(async move {
2276                let _ = subscriber.frames.send(frame).await;
2277            });
2278        }
2279        true
2280    }
2281
2282    fn remove_connection(&self, connection_id: ConnectionId) {
2283        self.0
2284            .lock()
2285            .unwrap_or_else(|p| p.into_inner())
2286            .subscribers
2287            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
2288    }
2289
2290    #[cfg(any(test, feature = "test-support"))]
2291    fn set_capacity(&self, capacity: usize) {
2292        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
2293    }
2294
2295    #[cfg(any(test, feature = "test-support"))]
2296    fn subscriber_count(&self) -> usize {
2297        self.0
2298            .lock()
2299            .unwrap_or_else(|p| p.into_inner())
2300            .subscribers
2301            .len()
2302    }
2303}
2304
2305/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
2306/// The terminal Error a lagged spawn subscriber receives after its queued events.
2307fn spawn_subscriber_lagged_frame(
2308    version: u8,
2309    corr: u64,
2310    first_undelivered: SpawnCursor,
2311) -> Result<Frame, String> {
2312    // The daemon authors this terminal Error, so it carries the daemon-origin bit.
2313    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
2314        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
2315        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
2316            .to_string(),
2317        detail: Some(serde_json::json!({
2318            "first_undelivered_cursor": first_undelivered
2319        })),
2320    })
2321    .map_err(|error| error.to_string())?;
2322    Frame::build_with_version(
2323        version,
2324        FrameType::Error,
2325        crate::router::daemon_error_flags(),
2326        0,
2327        0,
2328        corr,
2329        body,
2330    )
2331    .map_err(|error| error.to_string())
2332}
2333
2334pub trait ModuleProcessLiveness: Send + Sync {
2335    fn process_live(&self, module_id: &str) -> Option<bool>;
2336
2337    /// Whether the supervisor is replacing this module's process right now: an
2338    /// operator restart or reload, a health restart, or a crash respawn whose
2339    /// backoff is running. A module in that state is not live, but a consumer
2340    /// refused now should retry shortly rather than treat the target as gone.
2341    /// Stopped, failed, and disabled modules are not replacing.
2342    fn process_replacing(&self, _module_id: &str) -> bool {
2343        false
2344    }
2345}
2346
2347/// Shared process-liveness registry keyed by supervised `module_id`.
2348#[derive(Debug, Clone, Default)]
2349pub struct SupervisorProcessLiveness {
2350    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
2351}
2352
2353impl SupervisorProcessLiveness {
2354    pub fn new() -> Self {
2355        Self::default()
2356    }
2357
2358    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
2359        let mut snapshots = self
2360            .snapshots
2361            .lock()
2362            .unwrap_or_else(|poisoned| poisoned.into_inner());
2363        snapshots.insert(module_id, snapshot);
2364    }
2365
2366    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
2367        let mut snapshots = self
2368            .snapshots
2369            .lock()
2370            .unwrap_or_else(|poisoned| poisoned.into_inner());
2371        let is_current = snapshots
2372            .get(module_id)
2373            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
2374            .unwrap_or(false);
2375        if is_current {
2376            snapshots.remove(module_id);
2377        }
2378    }
2379}
2380
2381impl ModuleProcessLiveness for SupervisorProcessLiveness {
2382    fn process_live(&self, module_id: &str) -> Option<bool> {
2383        let snapshot = {
2384            let snapshots = self
2385                .snapshots
2386                .lock()
2387                .unwrap_or_else(|poisoned| poisoned.into_inner());
2388            snapshots.get(module_id).cloned()
2389        }?;
2390        let snapshot = snapshot
2391            .lock()
2392            .unwrap_or_else(|poisoned| poisoned.into_inner());
2393        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
2394    }
2395
2396    fn process_replacing(&self, module_id: &str) -> bool {
2397        let Some(snapshot) = self
2398            .snapshots
2399            .lock()
2400            .unwrap_or_else(|poisoned| poisoned.into_inner())
2401            .get(module_id)
2402            .cloned()
2403        else {
2404            return false;
2405        };
2406        let snapshot = snapshot
2407            .lock()
2408            .unwrap_or_else(|poisoned| poisoned.into_inner());
2409        snapshot.enabled
2410            && match snapshot.state {
2411                ModuleState::Restarting => true,
2412                ModuleState::Draining => snapshot.draining_to_replace,
2413                ModuleState::Starting
2414                | ModuleState::Running
2415                | ModuleState::Unresponsive
2416                | ModuleState::Stopped
2417                | ModuleState::Failed
2418                | ModuleState::Disabled => false,
2419            }
2420    }
2421}
2422
2423#[cfg(test)]
2424#[derive(Debug, Default)]
2425struct ReloadExitRecordGate {
2426    reached: tokio::sync::Notify,
2427    resume: tokio::sync::Notify,
2428}
2429
2430#[derive(Debug, Clone, Copy)]
2431enum RespawnKind {
2432    Spawn,
2433    Reload,
2434}
2435
2436#[derive(Debug, Clone)]
2437struct PendingRespawn {
2438    deadline: Instant,
2439    kind: RespawnKind,
2440    daemon_initiated: bool,
2441    approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
2442}
2443
2444impl PendingRespawn {
2445    fn failure_context(&self, runtime: &SupervisorRuntimeConfig) -> FailureContext {
2446        FailureContext {
2447            approval: self.approval.clone(),
2448            ..runtime.failure_context(self.daemon_initiated)
2449        }
2450    }
2451}
2452
2453type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
2454
2455#[derive(Debug, Clone)]
2456struct SupervisorRuntimeConfig {
2457    registry: Arc<Registry>,
2458    config_delivery: Option<config_delivery::ConfigDelivery>,
2459    /// Retained across an operation's deferred backoff, until its child starts.
2460    recovery_is_daemon: bool,
2461    start_requester: Option<Principal>,
2462    pending_activity_restart: bool,
2463    provenance_probe: ExecutableIdentityProbe,
2464    /// Restart operations hand their backoff to the loop, which keeps serving commands.
2465    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2466    /// A reload acknowledges completion only after its replacement registers.
2467    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2468    restart_policy: RestartPolicy,
2469    register_timeout: Duration,
2470    /// This module's RESOLVED drain budget: per-module config when present,
2471    /// else `default_drain_timeout`.
2472    drain_timeout: Duration,
2473    /// Shared with the status handle so the attested value changes atomically
2474    /// when a rescan updates the running drain policy.
2475    effective_drain_timeout: Arc<Mutex<Duration>>,
2476    /// The supervisor-wide fallback, kept so a configuration update that
2477    /// REMOVES the per-module override can re-resolve to it.
2478    default_drain_timeout: Duration,
2479    health: HealthConfig,
2480    host_health: HostHealth,
2481    connection_file_path: Option<PathBuf>,
2482    capture_logs_dir: Option<PathBuf>,
2483    forwarding: Option<Arc<ForwardingTable>>,
2484    /// The shared handle, so every spawn path (initial, restart, reload) records the
2485    /// reserved-module launch nonce the HELLO verifier checks against.
2486    supervisor_handle: Option<SupervisorHandle>,
2487    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
2488    /// status queries.
2489    ///
2490    /// One ring per module, held across every respawn. The lines explaining an exit
2491    /// are written BEFORE that exit, so a ring recreated per process would be empty
2492    /// exactly when it is asked for.
2493    stderr_ring: Arc<Mutex<StderrRing>>,
2494    terminal_ring: Arc<Mutex<TerminalRing>>,
2495    spawn_events: SpawnEventFeed,
2496    child_roster: ChildRoster,
2497    #[cfg(target_os = "linux")]
2498    cgroup_placement: Option<subc_cgroup::Placement>,
2499    #[cfg(test)]
2500    test_seed_stale_facts_before_enable_spawn: bool,
2501    #[cfg(test)]
2502    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2503}
2504
2505impl SupervisorRuntimeConfig {
2506    fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2507        FailureContext {
2508            daemon_initiated,
2509            approval: None,
2510            shutdown: self.child_roster.shutdown_flag(),
2511        }
2512    }
2513}
2514
2515#[derive(Debug, Clone, PartialEq, Eq)]
2516struct SupervisedConfiguration {
2517    spec: ModuleSpec,
2518    enabled: bool,
2519    health: HealthConfig,
2520    register_timeout: Duration,
2521    restart_policy: RestartPolicy,
2522}
2523
2524/// Shared daemon lookup table for supervised module handles.
2525///
2526/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
2527/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
2528/// launch nonces recorded at spawn are checked by the same daemon instance.
2529#[derive(Debug, Clone, Default)]
2530pub struct SupervisorHandle {
2531    #[cfg(test)]
2532    resource_commit_pause: Arc<Mutex<Option<resource_launch::ResourceCommitPause>>>,
2533    #[cfg(test)]
2534    registration_test_channels: CapturedRegistrationChannels,
2535    registration_service: Arc<Mutex<Option<registration::RegistrationService>>>,
2536    registration_connections: Arc<Mutex<HashMap<ConnectionId, Arc<RegistrationSpawn>>>>,
2537    registration_launches: Arc<Mutex<Vec<std::sync::Weak<RegistrationSpawn>>>>,
2538    gate_runtime: Arc<Mutex<Option<std::sync::Weak<crate::config_gate::runtime::RuntimeOwner>>>>,
2539    program_bindings: program_binding::ProgramBindings,
2540    #[cfg(test)]
2541    identity_sample_pause: Arc<Mutex<Option<IdentitySamplePause>>>,
2542    #[cfg(all(test, unix))]
2543    plain_identity_policy_pause: Arc<Mutex<Option<(String, IdentitySamplePause)>>>,
2544    host_health: HostHealth,
2545    restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2546    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2547    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2548    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2549    /// Module ids the supervisor has taken on. An id is added BEFORE the
2550    /// module's first process is spawned and removed only when the module
2551    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
2552    /// the keys of `modules`.
2553    ///
2554    /// `modules` cannot answer "is this module configured?" on its own: a
2555    /// [`SupervisedModule`] only exists once its process has been spawned, and
2556    /// a fast child can connect, register, sync its scopes and ask about them
2557    /// before the supervisor has inserted it. Answering "not configured" in that
2558    /// gap makes scope admission refuse with the terminal "will never sync"
2559    /// instead of the retryable "has not synced yet".
2560    configured_ids: Arc<Mutex<HashSet<String>>>,
2561    spawn_events: SpawnEventFeed,
2562    /// The current expected launch nonce for each reserved module_id. Set when the
2563    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
2564    /// non-reserved module never has an entry here and is never nonce-checked.
2565    /// Reserved module ids and the nonce that authorizes their next HELLO.
2566    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
2567    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
2568    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
2569    /// had NO entry and admitted anyone: the reservation protected the nonce
2570    /// holder, not the NAME (found live by CKCRED's canary probe registering
2571    /// against a reserved scratch id).
2572    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2573    /// Reservations installed only to prevent HELLO for a protocol:none process.
2574    /// Replacement by a subc process releases them; explicitly reserved ids are
2575    /// never placed in this set and still require an operator release.
2576    implicit_protocol_none: Arc<Mutex<HashSet<String>>>,
2577    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2578    ///
2579    /// This is deliberately in-memory only: subc is state-free across daemon
2580    /// restarts, and the tombstone only explains the hours-after-removal window
2581    /// while this executing daemon is still alive. Do not persist it in a store.
2582    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2583    /// The current launch nonce for every supervised spawn. This is separate from
2584    /// reserved_nonces because consumer route.open attestation applies to all spawned
2585    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2586    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2587    /// The launch each live launch nonce was minted for, keyed by nonce, with
2588    /// the launch's module id, so a nonce-attested consumer or registration
2589    /// resolves to that launch's authority record. An entry whose launch has
2590    /// ended stays while its nonce still authorizes, so a nonce whose record
2591    /// is gone is told apart from one that never had a record.
2592    nonce_launches: Arc<Mutex<NonceLaunches>>,
2593    /// Reserved namespace prefixes mapped to the supervised owner module whose
2594    /// current spawn nonce authorizes HELLO claims below the prefix.
2595    ///
2596    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2597    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2598    /// accidental collisions and lower-trust processes from squatting protected
2599    /// namespaces.
2600    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2601    /// Blue/green swaps in progress, by module id. An entry exists from just
2602    /// before the candidate process is spawned until the swap has failed, or
2603    /// has cut over and the old process is gone. While it exists, HELLO for the
2604    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2605    /// consumer attestation accepts both processes' nonces.
2606    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2607    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2608    promotion_observer: PromotionObserverSlot,
2609    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2610    /// this daemon-wide ordering, a rescan could retire or update a module while a
2611    /// concurrent reload still held its old handle and launch specification.
2612    operation_lock: Arc<AsyncMutex<()>>,
2613}
2614
2615#[cfg(test)]
2616#[derive(Debug)]
2617struct IdentitySamplePause {
2618    reached: std::sync::mpsc::Sender<()>,
2619    release: std::sync::mpsc::Receiver<()>,
2620}
2621
2622/// Told when a swap has promoted its candidate to be the module's active
2623/// registration.
2624///
2625/// An ordinary HELLO runs the control plane's registration side effects (the
2626/// capability cache, the deny census, the requirement recompute) as it
2627/// registers. A swap candidate's HELLO does not, because it is not routable;
2628/// promotion is when those must run instead, and promotion happens in the
2629/// supervisor, which has no other way into the control handler.
2630pub(crate) trait SwapPromotionObserver: Send + Sync {
2631    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2632}
2633
2634/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2635/// control handler) owns this handle, so a strong reference back would be a
2636/// cycle that keeps both alive.
2637#[derive(Clone, Default)]
2638struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2639
2640impl fmt::Debug for PromotionObserverSlot {
2641    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2642        f.write_str("PromotionObserverSlot")
2643    }
2644}
2645
2646/// The nonces of one open swap.
2647#[derive(Debug, Clone)]
2648struct OpenSwap {
2649    /// Legacy candidate token. A gated candidate has an empty lifecycle
2650    /// placeholder instead; only its own channel admits its HELLO.
2651    candidate_nonce: String,
2652    /// Incumbent token accepted before cutover, then cleared by promotion even
2653    /// while the incumbent's process is still draining.
2654    incumbent_nonce: Option<String>,
2655    /// Set once a HELLO has been admitted with the swap token, so the token
2656    /// admits one registration and cannot be replayed after cutover empties
2657    /// the candidate slot.
2658    candidate_admitted: bool,
2659}
2660
2661/// What the swap gate says about a HELLO. See
2662/// [`SupervisorHandle::swap_hello_admission`].
2663#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2664pub(crate) enum SwapHelloAdmission {
2665    /// No swap is open for the id (or the HELLO carries the incumbent's own
2666    /// nonce); the ordinary gates decide.
2667    NotSwapping,
2668    /// The HELLO carries the swap token: register it into the candidate slot.
2669    Candidate,
2670    /// A swap is open and the HELLO carries a nonce the supervisor did not
2671    /// mint for this id, no nonce, or a token already used.
2672    Refused,
2673    /// The id belongs to a supervised `protocol: none` module. Such a module
2674    /// never registers, so no HELLO may claim its id, with or without a nonce.
2675    ProtocolNone,
2676    /// The id is gated. Its supervised child registers only over the
2677    /// registration channel it inherited at launch, never by HELLO alone.
2678    ChannelRequired,
2679}
2680
2681#[derive(Debug, Clone, PartialEq, Eq)]
2682pub(crate) enum ReservedHelloRejection {
2683    Exact {
2684        module_id: String,
2685    },
2686    Prefix {
2687        prefix: String,
2688        owner_module_id: String,
2689    },
2690}
2691
2692/// Identity facts established by the reserved-module policy for one HELLO.
2693#[derive(Debug, Default)]
2694pub(crate) struct ReservedHelloVerification {
2695    pub(crate) nonce_verified: bool,
2696    pub(crate) prefix_owner: Option<String>,
2697}
2698
2699#[derive(Debug)]
2700struct DaemonRestartIntent {
2701    expires: Option<Duration>,
2702    window: Duration,
2703}
2704
2705impl Default for DaemonRestartIntent {
2706    fn default() -> Self {
2707        Self {
2708            expires: None,
2709            window: Duration::from_secs(120),
2710        }
2711    }
2712}
2713
2714impl SupervisorHandle {
2715    pub(crate) fn module_is_gated(&self, module: &str) -> bool {
2716        let runtime = {
2717            let Ok(runtime) = self.gate_runtime.lock() else {
2718                return true;
2719            };
2720            runtime.clone()
2721        };
2722        runtime.is_some_and(|runtime| {
2723            runtime
2724                .upgrade()
2725                .is_none_or(|owner| owner.module_is_gated(module))
2726        })
2727    }
2728    pub(crate) fn announce_daemon_restart(&self) -> u64 {
2729        let mut intent = self
2730            .restart_intent
2731            .lock()
2732            .unwrap_or_else(|p| p.into_inner());
2733        intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2734        unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2735    }
2736
2737    #[cfg(feature = "test-support")]
2738    pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2739        self.restart_intent
2740            .lock()
2741            .unwrap_or_else(|p| p.into_inner())
2742            .window = window;
2743    }
2744
2745    #[cfg(unix)]
2746    fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2747        let expiry = self
2748            .restart_intent
2749            .lock()
2750            .unwrap_or_else(|p| p.into_inner())
2751            .expires
2752            .take();
2753        if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2754            RouteCloseReason::Restart
2755        } else {
2756            RouteCloseReason::Disable
2757        }
2758    }
2759
2760    pub fn new() -> Self {
2761        Self::default()
2762    }
2763
2764    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2765        self.spawn_events.snapshot()
2766    }
2767
2768    /// Subscribe before reading the bound process. A coalesced wake is enough:
2769    /// readers compare its exact generation and PID, not the last event kind.
2770    pub(crate) fn subscribe_process_changes(
2771        &self,
2772        module: &str,
2773    ) -> tokio::sync::watch::Receiver<u64> {
2774        let mut state = self
2775            .spawn_events
2776            .0
2777            .lock()
2778            .unwrap_or_else(|p| p.into_inner());
2779        state
2780            .process_changes
2781            .retain(|_, changed| changed.receiver_count() > 0);
2782        let seq = state.seq;
2783        state
2784            .process_changes
2785            .entry(module.into())
2786            .or_insert_with(|| tokio::sync::watch::channel(seq).0)
2787            .subscribe()
2788    }
2789
2790    pub(crate) fn current_spawn(&self, module: &str) -> Option<LiveSpawn> {
2791        self.spawn_events
2792            .0
2793            .lock()
2794            .unwrap_or_else(|p| p.into_inner())
2795            .live
2796            .get(module)
2797            .cloned()
2798    }
2799
2800    pub(crate) fn subscribe_spawns(
2801        &self,
2802        connection_id: ConnectionId,
2803        corr: u64,
2804        version: u8,
2805        since: Option<SpawnCursor>,
2806        sink: FrameSink,
2807    ) -> Result<(), SpawnSubscribeRefusal> {
2808        self.spawn_events
2809            .subscribe(connection_id, corr, version, since, sink)
2810    }
2811
2812    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2813        self.spawn_events.cancel(connection_id, corr)
2814    }
2815
2816    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2817        self.spawn_events.remove_connection(connection_id);
2818    }
2819
2820    #[cfg(any(test, feature = "test-support"))]
2821    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2822        assert!(capacity > 0, "spawn event capacity must be non-zero");
2823        self.spawn_events.set_capacity(capacity);
2824    }
2825
2826    #[cfg(any(test, feature = "test-support"))]
2827    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2828        self.spawn_events.subscriber_count()
2829    }
2830
2831    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2832    /// a respawn invalidates stale consumer identities.
2833    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2834        self.spawn_nonces
2835            .lock()
2836            .unwrap_or_else(|poisoned| poisoned.into_inner())
2837            .insert(module_id.to_string(), nonce);
2838        self.refresh_program_binding_nonces(module_id);
2839    }
2840
2841    /// Record the launch nonce expected from the next HELLO for a reserved module,
2842    /// replacing any prior nonce (a respawn invalidates the previous one).
2843    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2844        {
2845            let mut reserved = self
2846                .reserved_nonces
2847                .lock()
2848                .unwrap_or_else(|p| p.into_inner());
2849            self.implicit_protocol_none
2850                .lock()
2851                .unwrap_or_else(|p| p.into_inner())
2852                .remove(module_id);
2853            reserved.insert(module_id.to_string(), Some(nonce));
2854        }
2855        self.refresh_program_binding_nonces(module_id);
2856    }
2857
2858    /// Record namespace prefixes owned by a supervised module.
2859    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2860        let mut owners = self
2861            .reserved_prefix_owners
2862            .lock()
2863            .unwrap_or_else(|poisoned| poisoned.into_inner());
2864        owners.retain(|_, owner| owner != owner_module_id);
2865        for prefix in prefixes {
2866            owners.insert(prefix.clone(), owner_module_id.to_string());
2867        }
2868    }
2869
2870    /// The launch nonce most recently minted for a module's spawn, if any.
2871    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2872        if self.module_is_gated(module_id) {
2873            return None;
2874        }
2875        self.spawn_nonces
2876            .lock()
2877            .unwrap_or_else(|poisoned| poisoned.into_inner())
2878            .get(module_id)
2879            .cloned()
2880    }
2881
2882    #[cfg(all(test, unix))]
2883    pub(crate) fn legacy_nonce_present_for_test(&self, module: &str) -> bool {
2884        self.spawn_nonces.lock().unwrap().contains_key(module)
2885    }
2886
2887    pub(crate) fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2888        // Prefix ownership follows staged configuration, not the running
2889        // child's launch settings. A respawn may still carry an older spec.
2890        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2891        let protocol = self
2892            .get(&spec.module_id)
2893            .and_then(|module| {
2894                let snapshot = module.inner.snapshot.lock().ok()?;
2895                snapshot
2896                    .process_alive
2897                    .then_some(snapshot.spawned_protocol.unwrap_or(spec.protocol))
2898            })
2899            .unwrap_or(spec.protocol);
2900        self.apply_identity_policy(spec, protocol);
2901    }
2902
2903    #[cfg(all(test, unix))]
2904    pub(crate) fn pause_plain_identity_policy_for_test(
2905        &self,
2906        module: &str,
2907    ) -> (std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>) {
2908        let (reached, waiting) = std::sync::mpsc::channel();
2909        let (release, paused) = std::sync::mpsc::channel();
2910        *self.plain_identity_policy_pause.lock().unwrap() = Some((
2911            module.to_owned(),
2912            IdentitySamplePause {
2913                reached,
2914                release: paused,
2915            },
2916        ));
2917        (waiting, release)
2918    }
2919
2920    fn apply_identity_policy(&self, spec: &ModuleSpec, protocol: ModuleProtocol) {
2921        let channel_only = self.module_is_gated(&spec.module_id);
2922        let mut current_nonces = self
2923            .spawn_nonces
2924            .lock()
2925            .unwrap_or_else(|poisoned| poisoned.into_inner());
2926        if protocol == ModuleProtocol::None || channel_only {
2927            current_nonces.remove(&spec.module_id);
2928        }
2929        let spawn_nonce = current_nonces.get(&spec.module_id).cloned();
2930        #[cfg(test)]
2931        if let Some(pause) = self.identity_sample_pause.lock().unwrap().take() {
2932            pause.reached.send(()).unwrap();
2933            pause.release.recv().unwrap();
2934        }
2935        let mut reserved_nonces = self
2936            .reserved_nonces
2937            .lock()
2938            .unwrap_or_else(|poisoned| poisoned.into_inner());
2939        let mut implicit = self
2940            .implicit_protocol_none
2941            .lock()
2942            .unwrap_or_else(|p| p.into_inner());
2943        if spec.reserved {
2944            implicit.remove(&spec.module_id);
2945        } else if protocol == ModuleProtocol::None {
2946            if !reserved_nonces.contains_key(&spec.module_id) {
2947                implicit.insert(spec.module_id.clone());
2948            }
2949        } else if implicit.remove(&spec.module_id) {
2950            reserved_nonces.remove(&spec.module_id);
2951        }
2952        if spec.reserved || protocol == ModuleProtocol::None {
2953            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2954            // reserved name whose module has never spawned has no legitimate
2955            // holder, and the entry's absence is what used to leave the name
2956            // open to the first claimant.
2957            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2958        }
2959        drop(implicit);
2960        drop(reserved_nonces);
2961        drop(current_nonces);
2962        self.refresh_program_binding_nonces(&spec.module_id);
2963        // A later unreserved declaration must not silently unreserve an id that
2964        // was retained after its reserved configuration was removed. The explicit
2965        // release ceremony is the only operation that retires that gate.
2966        self.removal_tombstones
2967            .lock()
2968            .unwrap_or_else(|poisoned| poisoned.into_inner())
2969            .remove(&spec.module_id);
2970    }
2971
2972    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2973    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2974    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2975    /// with no matching prefix are always authorized.
2976    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2977        self.reserved_hello_verification(module_id, presented)
2978            .is_ok()
2979    }
2980
2981    /// Whether HELLO passed a nonce check that was actually required by reserved
2982    /// module policy, or the refusal explaining why a required check failed.
2983    pub(crate) fn reserved_hello_verification(
2984        &self,
2985        module_id: &str,
2986        presented: Option<&str>,
2987    ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2988        if self.module_is_gated(module_id) {
2989            return Err(ReservedHelloRejection::Exact {
2990                module_id: module_id.into(),
2991            });
2992        }
2993        self.hello_verification(module_id, presented, false)
2994    }
2995
2996    pub(crate) fn channel_hello_verification(
2997        &self,
2998        module_id: &str,
2999    ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
3000        if self
3001            .get(module_id)
3002            .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
3003        {
3004            return Err(ReservedHelloRejection::Exact {
3005                module_id: module_id.into(),
3006            });
3007        }
3008        let mut verification = self.hello_verification(module_id, None, true)?;
3009        verification.nonce_verified = true;
3010        Ok(verification)
3011    }
3012
3013    pub(crate) fn prefix_claim_authorized(&self, owner: &str, module_id: &str) -> bool {
3014        self.prefix_claim_publication(owner, module_id).is_some()
3015    }
3016
3017    #[cfg(all(test, unix))]
3018    pub(crate) fn prefix_publication_locked_for_test(&self) -> bool {
3019        matches!(
3020            self.reserved_prefix_owners.try_lock(),
3021            Err(std::sync::TryLockError::WouldBlock)
3022        )
3023    }
3024
3025    // Keep this guard through registry/forwarding publication. Rescans replace
3026    // prefix owners without stopping their launches, so the spawn guard alone
3027    // cannot fence a reservation transfer.
3028    pub(crate) fn prefix_claim_publication(
3029        &self,
3030        owner: &str,
3031        module_id: &str,
3032    ) -> Option<std::sync::MutexGuard<'_, HashMap<String, String>>> {
3033        if subc_protocol::manifest::validate_module_id(module_id).is_err()
3034            || self.module_is_gated(module_id)
3035            || self
3036                .reserved_nonces
3037                .lock()
3038                .unwrap_or_else(|p| p.into_inner())
3039                .contains_key(module_id)
3040        {
3041            return None;
3042        }
3043        let owners = self
3044            .reserved_prefix_owners
3045            .lock()
3046            .unwrap_or_else(|p| p.into_inner());
3047        let authorized = owners
3048            .iter()
3049            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
3050            .max_by_key(|(prefix, _)| prefix.len())
3051            .is_some_and(|(prefix, holder)| holder == owner && module_id.len() > prefix.len());
3052        authorized.then_some(owners)
3053    }
3054
3055    fn hello_verification(
3056        &self,
3057        module_id: &str,
3058        presented: Option<&str>,
3059        channel: bool,
3060    ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
3061        let nonces = self
3062            .reserved_nonces
3063            .lock()
3064            .unwrap_or_else(|poisoned| poisoned.into_inner());
3065        if let Some(expected) = nonces.get(module_id) {
3066            // `None` = reserved with no legitimate holder: refuse every
3067            // presentation, because no process can hold a nonce that was never
3068            // minted. Only a real minted nonce admits, in constant time.
3069            let authorized = channel
3070                || match expected {
3071                    Some(expected) => presented.is_some_and(|p| {
3072                        !p.is_empty() && constant_time_eq(expected.as_bytes(), p.as_bytes())
3073                    }),
3074                    None => false,
3075                };
3076            if authorized {
3077                return Ok(ReservedHelloVerification {
3078                    nonce_verified: true,
3079                    prefix_owner: None,
3080                });
3081            }
3082            return Err(ReservedHelloRejection::Exact {
3083                module_id: module_id.to_string(),
3084            });
3085        }
3086        drop(nonces);
3087
3088        let matched_prefix = self
3089            .reserved_prefix_owners
3090            .lock()
3091            .unwrap_or_else(|poisoned| poisoned.into_inner())
3092            .iter()
3093            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
3094            .max_by_key(|(prefix, _)| prefix.len())
3095            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
3096        let Some((prefix, owner_module_id)) = matched_prefix else {
3097            return Ok(ReservedHelloVerification::default());
3098        };
3099
3100        if module_id.len() == prefix.len()
3101            || (!channel
3102                && (self.module_is_gated(&owner_module_id) || self.protocol_none(&owner_module_id)))
3103        {
3104            return Err(ReservedHelloRejection::Prefix {
3105                prefix,
3106                owner_module_id,
3107            });
3108        }
3109
3110        let authorized = if channel {
3111            owner_module_id == module_id
3112        } else {
3113            presented.is_some_and(|presented| {
3114                self.spawn_nonces
3115                .lock()
3116                .unwrap_or_else(|poisoned| poisoned.into_inner())
3117                .get(&owner_module_id)
3118                .is_some_and(|expected| !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes()))
3119                // While the owner is being swapped, children started by
3120                // either of its two processes hold that process's nonce.
3121                || self.swap_nonce_matches(&owner_module_id, presented)
3122            })
3123        };
3124        if authorized {
3125            Ok(ReservedHelloVerification {
3126                nonce_verified: true,
3127                prefix_owner: (owner_module_id != module_id).then_some(owner_module_id),
3128            })
3129        } else {
3130            Err(ReservedHelloRejection::Prefix {
3131                prefix,
3132                owner_module_id,
3133            })
3134        }
3135    }
3136
3137    /// Whether a consumer connection proved it came from a daemon-spawned module.
3138    ///
3139    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
3140    /// accepted only for module ids the supervisor has spawned.
3141    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
3142        if self.module_is_gated(module_id) || self.protocol_none(module_id) {
3143            return false;
3144        }
3145        let nonces = self
3146            .spawn_nonces
3147            .lock()
3148            .unwrap_or_else(|poisoned| poisoned.into_inner());
3149        let current = nonces.get(module_id).is_some_and(|expected| {
3150            !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes())
3151        });
3152        drop(nonces);
3153        // Before cutover both launches may attest. Promotion removes the
3154        // incumbent's token even if its process still needs time to drain.
3155        current || self.swap_nonce_matches(module_id, presented)
3156    }
3157
3158    /// Whether `presented` is either nonce of an open swap for `module_id`.
3159    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
3160        if self.module_is_gated(module_id) || self.protocol_none(module_id) || presented.is_empty()
3161        {
3162            return false;
3163        }
3164        let swaps = self
3165            .swaps
3166            .lock()
3167            .unwrap_or_else(|poisoned| poisoned.into_inner());
3168        swaps.get(module_id).is_some_and(|swap| {
3169            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
3170                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
3171                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
3172                })
3173        })
3174    }
3175
3176    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
3177    /// Called before the candidate process exists.
3178    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
3179        let incumbent_nonce = self
3180            .spawn_nonces
3181            .lock()
3182            .unwrap_or_else(|poisoned| poisoned.into_inner())
3183            .get(module_id)
3184            .cloned();
3185        self.swaps
3186            .lock()
3187            .unwrap_or_else(|poisoned| poisoned.into_inner())
3188            .insert(
3189                module_id.to_string(),
3190                OpenSwap {
3191                    candidate_nonce,
3192                    incumbent_nonce,
3193                    candidate_admitted: false,
3194                },
3195            );
3196        self.refresh_program_binding_nonces(module_id);
3197    }
3198
3199    /// Close the swap for `module_id`, releasing whichever nonce is no longer
3200    /// the module's recorded one.
3201    pub(crate) fn close_swap(&self, module_id: &str) {
3202        self.swaps
3203            .lock()
3204            .unwrap_or_else(|poisoned| poisoned.into_inner())
3205            .remove(module_id);
3206        self.refresh_program_binding_nonces(module_id);
3207    }
3208
3209    /// Install the observer told about swap promotions, replacing any earlier
3210    /// one.
3211    pub(crate) fn set_swap_promotion_observer(
3212        &self,
3213        observer: std::sync::Weak<dyn SwapPromotionObserver>,
3214    ) {
3215        *self
3216            .promotion_observer
3217            .0
3218            .lock()
3219            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
3220    }
3221
3222    /// Tell the installed observer, if it is still alive, that a swap promoted
3223    /// `registration`.
3224    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
3225        let observer = self
3226            .promotion_observer
3227            .0
3228            .lock()
3229            .unwrap_or_else(|poisoned| poisoned.into_inner())
3230            .as_ref()
3231            .and_then(std::sync::Weak::upgrade);
3232        if let Some(observer) = observer {
3233            observer.swap_promoted(registration);
3234        }
3235    }
3236
3237    /// Whether a swap is open for `module_id`.
3238    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
3239        self.swaps
3240            .lock()
3241            .unwrap_or_else(|poisoned| poisoned.into_inner())
3242            .contains_key(module_id)
3243    }
3244
3245    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
3246    /// respawn would, once cutover has made the candidate the module's process.
3247    /// Promotion revokes the incumbent's tokens; the swap record stays open
3248    /// only for lifecycle cleanup while the old process drains.
3249    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
3250        let candidate_nonce = self
3251            .swaps
3252            .lock()
3253            .unwrap_or_else(|poisoned| poisoned.into_inner())
3254            .get_mut(module_id)
3255            .map(|swap| {
3256                swap.incumbent_nonce = None;
3257                swap.candidate_nonce.clone()
3258            });
3259        self.spawn_nonces
3260            .lock()
3261            .unwrap_or_else(|p| p.into_inner())
3262            .remove(module_id);
3263        {
3264            let mut nonces = self
3265                .reserved_nonces
3266                .lock()
3267                .unwrap_or_else(|p| p.into_inner());
3268            if reserved || nonces.contains_key(module_id) {
3269                nonces.insert(module_id.into(), None);
3270            }
3271        }
3272        if self.module_is_gated(module_id) {
3273            return;
3274        }
3275        let Some(nonce) = candidate_nonce.filter(|nonce| !nonce.is_empty()) else {
3276            return;
3277        };
3278        self.set_spawn_nonce(module_id, nonce.clone());
3279        if reserved {
3280            self.set_reserved_nonce(module_id, nonce);
3281        }
3282    }
3283
3284    /// The swap gate for a HELLO claiming `module_id`.
3285    ///
3286    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
3287    /// presents the candidate nonce, which the reserved gate (holding the
3288    /// incumbent's nonce) would refuse as `reserved_module` before swap
3289    /// admission was ever reached. And it applies to unreserved ids too: for an
3290    /// unreserved id the only thing that ever stopped a second process claiming
3291    /// a live id was the `duplicate_module_id` refusal, which is exactly the
3292    /// refusal a swap lifts for its candidate.
3293    ///
3294    /// The incumbent's own nonce falls through to the ordinary gates, which
3295    /// treat it as they always have (a live incumbent is refused as a
3296    /// duplicate). Anything else while a swap is open is refused, including an
3297    /// absent nonce.
3298    pub(crate) fn swap_hello_admission(
3299        &self,
3300        module_id: &str,
3301        presented: Option<&str>,
3302    ) -> SwapHelloAdmission {
3303        if self.protocol_none(module_id) {
3304            return SwapHelloAdmission::ProtocolNone;
3305        }
3306        if self.module_is_gated(module_id) {
3307            return SwapHelloAdmission::ChannelRequired;
3308        }
3309        let swaps = self
3310            .swaps
3311            .lock()
3312            .unwrap_or_else(|poisoned| poisoned.into_inner());
3313        let Some(swap) = swaps.get(module_id) else {
3314            return SwapHelloAdmission::NotSwapping;
3315        };
3316        let Some(presented) = presented.filter(|nonce| !nonce.is_empty()) else {
3317            return SwapHelloAdmission::Refused;
3318        };
3319        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
3320            return if swap.candidate_admitted {
3321                SwapHelloAdmission::Refused
3322            } else {
3323                SwapHelloAdmission::Candidate
3324            };
3325        }
3326        if swap
3327            .incumbent_nonce
3328            .as_deref()
3329            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
3330        {
3331            return SwapHelloAdmission::NotSwapping;
3332        }
3333        SwapHelloAdmission::Refused
3334    }
3335
3336    fn protocol_none(&self, module: &str) -> bool {
3337        self.get(module)
3338            .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
3339    }
3340
3341    /// Exclude swap closure through the short, synchronous candidate publication.
3342    pub(crate) fn with_swap_candidate_admission<T>(
3343        &self,
3344        module_id: &str,
3345        publish: impl FnOnce() -> (T, bool),
3346    ) -> Option<T> {
3347        let mut swaps = self.swaps.lock().unwrap_or_else(|p| p.into_inner());
3348        let swap = swaps
3349            .get_mut(module_id)
3350            .filter(|swap| !swap.candidate_admitted)?;
3351        let (result, admitted) = publish();
3352        swap.candidate_admitted = admitted;
3353        Some(result)
3354    }
3355
3356    /// Test/support lookup for the current launch nonce of a supervised spawn.
3357    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3358        self.spawn_nonce(module_id)
3359    }
3360
3361    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
3362    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3363        if self.module_is_gated(module_id) {
3364            return None;
3365        }
3366        self.reserved_nonces
3367            .lock()
3368            .unwrap_or_else(|poisoned| poisoned.into_inner())
3369            .get(module_id)
3370            .cloned()
3371            .flatten()
3372    }
3373
3374    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
3375        // Normally already marked before the process was spawned; marking here
3376        // too keeps `configured_ids` a superset of the roster for any caller
3377        // that inserts a module directly.
3378        self.mark_configured(module.module_id());
3379        let mut modules = self
3380            .modules
3381            .lock()
3382            .unwrap_or_else(|poisoned| poisoned.into_inner());
3383        modules.insert(module.module_id().to_string(), module)
3384    }
3385
3386    /// Record that the supervisor has taken on `module_id`. Called before the
3387    /// module's first process is spawned, so that by the time that process can
3388    /// register, [`Self::is_configured`] already answers true.
3389    fn mark_configured(&self, module_id: &str) {
3390        self.update_configured_ids(|ids| {
3391            ids.insert(module_id.to_string());
3392        });
3393    }
3394
3395    fn publish_configured_ids(&self, ids: Option<&HashSet<String>>) {
3396        let owner = self
3397            .gate_runtime
3398            .lock()
3399            .unwrap_or_else(|p| p.into_inner())
3400            .clone()
3401            .and_then(|owner| owner.upgrade());
3402        if let Some(owner) = owner {
3403            owner.observe_supervisor_configured_ids(ids, Arc::downgrade(&self.configured_ids));
3404        }
3405    }
3406
3407    fn update_configured_ids(&self, update: impl FnOnce(&mut HashSet<String>)) {
3408        match self.configured_ids.lock() {
3409            Ok(mut ids) => {
3410                update(&mut ids);
3411                // Publish while this roster mutation is serialized, so an
3412                // older snapshot can never overwrite a later configured id.
3413                self.publish_configured_ids(Some(&ids));
3414            }
3415            Err(poisoned) => {
3416                update(&mut poisoned.into_inner());
3417                self.publish_configured_ids(None);
3418            }
3419        }
3420    }
3421
3422    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
3423    /// before it was ever put on the roster. A module already on the roster
3424    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
3425    fn unmark_configured_unless_rostered(&self, module_id: &str) {
3426        let modules = self
3427            .modules
3428            .lock()
3429            .unwrap_or_else(|poisoned| poisoned.into_inner());
3430        if !modules.contains_key(module_id) {
3431            self.update_configured_ids(|ids| {
3432                ids.remove(module_id);
3433            });
3434        }
3435    }
3436
3437    /// Whether `module_id` is a module this daemon supervises: on the roster,
3438    /// or about to be (its process is being spawned right now).
3439    ///
3440    /// This, not `get(..).is_some()`, is what "the owner is configured" means
3441    /// for scopes: a supervised module's process can register and sync before
3442    /// [`Self::get`] can return it, and in that window it is still a module
3443    /// that will sync, not one that never will.
3444    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
3445        self.configured_ids
3446            .lock()
3447            .unwrap_or_else(|poisoned| poisoned.into_inner())
3448            .contains(module_id)
3449    }
3450
3451    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
3452        let modules = self
3453            .modules
3454            .lock()
3455            .unwrap_or_else(|poisoned| poisoned.into_inner());
3456        modules.get(module_id).cloned()
3457    }
3458
3459    pub(crate) fn record_late_health_answer(
3460        &self,
3461        module_id: &str,
3462        latency_ms: u64,
3463    ) -> Result<bool, SuperviseError> {
3464        let Some(module) = self.get(module_id) else {
3465            return Ok(false);
3466        };
3467        self.host_health.record_late_module(module_id);
3468        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
3469            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
3470            state.health.last_late_answer_latency_ms = Some(latency_ms);
3471            // A late answer is an answer: the module served the probe, just past
3472            // the deadline. Leaving the miss streak in place while logging
3473            // "proves the module is alive" is how a CPU-starved module that
3474            // answers every probe a few seconds late still marches to the
3475            // threshold and gets killed — the exact kill class `NoAnswer` is
3476            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
3477            // is degradation, and degradation reports; it does not restart.
3478            state.health.consecutive_failures = 0;
3479        })?;
3480        Ok(true)
3481    }
3482
3483    pub(crate) fn spawn_host_health_monitor(&self) -> JoinHandle<()> {
3484        self.host_health.spawn_monitor()
3485    }
3486
3487    /// Arm the one-shot marker for the module process that this caller
3488    /// deliberately initiated severance against. Generic connection teardown
3489    /// must not call this:
3490    /// a surviving process would otherwise retain an exemption for a later
3491    /// genuine crash.
3492    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
3493        let Some(module) = self.get(module_id) else {
3494            return Ok(false);
3495        };
3496        let snapshot = lock_snapshot(&module.inner.snapshot)?;
3497        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
3498            return Ok(false);
3499        };
3500        drop(snapshot);
3501        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
3502    }
3503
3504    pub fn list(&self) -> Vec<SupervisedModule> {
3505        let modules = self
3506            .modules
3507            .lock()
3508            .unwrap_or_else(|poisoned| poisoned.into_inner());
3509        let mut modules = modules.values().cloned().collect::<Vec<_>>();
3510        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
3511        modules
3512    }
3513
3514    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
3515        self.activity_images
3516            .lock()
3517            .unwrap_or_else(|p| p.into_inner())
3518            .remove(module_id);
3519        self.spawn_nonces
3520            .lock()
3521            .unwrap_or_else(|poisoned| poisoned.into_inner())
3522            .remove(module_id);
3523        self.close_swap(module_id);
3524        let mut reserved_nonces = self
3525            .reserved_nonces
3526            .lock()
3527            .unwrap_or_else(|poisoned| poisoned.into_inner());
3528        if reserved_nonces.contains_key(module_id) {
3529            // The old nonce must die with the removed process, but the exact-id
3530            // gate remains until an operator explicitly releases it.
3531            reserved_nonces.insert(module_id.to_string(), None);
3532        }
3533        drop(reserved_nonces);
3534        self.reserved_prefix_owners
3535            .lock()
3536            .unwrap_or_else(|poisoned| poisoned.into_inner())
3537            .retain(|_, owner| owner != module_id);
3538        let removed = self
3539            .modules
3540            .lock()
3541            .unwrap_or_else(|poisoned| poisoned.into_inner())
3542            .remove(module_id);
3543        self.update_configured_ids(|ids| {
3544            ids.remove(module_id);
3545        });
3546        self.remove_program_binding_records(module_id);
3547        removed
3548    }
3549
3550    /// Remember a module removed by a non-preview rescan so route.open can
3551    /// distinguish that intentional removal from an unknown id.
3552    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
3553        self.removal_tombstones
3554            .lock()
3555            .unwrap_or_else(|poisoned| poisoned.into_inner())
3556            .insert(module_id.to_string(), unix_ms_now());
3557    }
3558
3559    /// Return how long ago a rescan removed this module in milliseconds.
3560    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
3561        self.removal_tombstones
3562            .lock()
3563            .unwrap_or_else(|poisoned| poisoned.into_inner())
3564            .get(module_id)
3565            .copied()
3566            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
3567    }
3568
3569    /// Retire a reserved-id gate only after its module has left supervision.
3570    ///
3571    /// A retained gate has no live nonce (`None`), so releasing any other entry
3572    /// would weaken a currently configured or otherwise active reservation.
3573    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
3574        if self.get(module_id).is_some() {
3575            return false;
3576        }
3577        let mut reserved_nonces = self
3578            .reserved_nonces
3579            .lock()
3580            .unwrap_or_else(|poisoned| poisoned.into_inner());
3581        if !matches!(reserved_nonces.get(module_id), Some(None)) {
3582            return false;
3583        }
3584        reserved_nonces.remove(module_id);
3585        self.implicit_protocol_none
3586            .lock()
3587            .unwrap_or_else(|p| p.into_inner())
3588            .remove(module_id);
3589        true
3590    }
3591
3592    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
3593        Arc::clone(&self.operation_lock)
3594    }
3595}
3596
3597/// Process supervisor for subc-owned singleton modules.
3598#[derive(Debug, Clone)]
3599pub struct Supervisor {
3600    registry: Arc<Registry>,
3601    config_delivery: Option<config_delivery::ConfigDelivery>,
3602    restart_policy: RestartPolicy,
3603    drain_timeout: Duration,
3604    connection_file_path: Option<PathBuf>,
3605    capture_logs_dir: Option<PathBuf>,
3606    forwarding: Option<Arc<ForwardingTable>>,
3607    process_liveness: Arc<SupervisorProcessLiveness>,
3608    supervisor_handle: Option<SupervisorHandle>,
3609    health: HealthConfig,
3610    host_health: HostHealth,
3611    daemon_start_clock: crate::clock::StartClock,
3612    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
3613    spawn_events: SpawnEventFeed,
3614    provenance_probe: ExecutableIdentityProbe,
3615    /// Every process spawned through this supervisor (and its clones) and not
3616    /// yet reaped, so daemon shutdown can end them.
3617    child_roster: ChildRoster,
3618    #[cfg(target_os = "linux")]
3619    cgroup_placement: Option<subc_cgroup::Placement>,
3620    #[cfg(test)]
3621    test_after_first_spawn: AfterFirstSpawnHook,
3622}
3623
3624/// A roster entry whose monitor and first process have not started yet.
3625/// Keeping the command receiver here lets discovery serve status immediately
3626/// while start/stop requests wait until bootstrap has published the connection file.
3627pub(crate) struct RegisteredConfiguredModule {
3628    spec: ModuleSpec,
3629    runtime: SupervisorRuntimeConfig,
3630    module: SupervisedModule,
3631    commands: mpsc::Receiver<SupervisorCommand>,
3632}
3633
3634/// Test-only hook run on the path that takes on a new module, right after its
3635/// first `spawn_child` returns (the process exists and could already be
3636/// registering) and before that process is handed to the module's supervise
3637/// loop and put on the roster. Lets a test observe what a fast child would see
3638/// in that window without racing a real one.
3639#[cfg(test)]
3640#[derive(Clone, Default)]
3641struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
3642
3643#[cfg(test)]
3644type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
3645
3646#[cfg(test)]
3647impl fmt::Debug for AfterFirstSpawnHook {
3648    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3649        f.write_str("AfterFirstSpawnHook")
3650    }
3651}
3652
3653#[cfg(test)]
3654impl AfterFirstSpawnHook {
3655    fn run(&self, module_id: &str) {
3656        if let Some(hook) = &self.0 {
3657            hook(module_id);
3658        }
3659    }
3660}
3661
3662impl Supervisor {
3663    #[cfg(all(test, target_os = "linux"))]
3664    pub(crate) fn set_admission_hook_for_test(
3665        &self,
3666        hook: impl Fn(&str, &'static str, Option<u32>) -> bool + Send + Sync + 'static,
3667    ) {
3668        self.child_roster.set_admission_hook_for_test(hook);
3669    }
3670
3671    #[cfg(test)]
3672    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
3673        let supervisor = Self::new(registry, policy);
3674        #[cfg(target_os = "macos")]
3675        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
3676        supervisor
3677    }
3678    /// Verify the trampoline once when configured. Missing private OS support
3679    /// refuses every macOS launch by name but does not stop the daemon's control
3680    /// server. Embedders must explicitly provide a binary with the subc-os hidden
3681    /// entry point; the library must not exec an arbitrary hosting program.
3682    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
3683        let path = path.into();
3684        #[cfg(target_os = "macos")]
3685        {
3686            let result = probe_privacy_trampoline(&path).map(|()| path);
3687            if let Err(cause) = &result {
3688                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
3689            }
3690            self.child_roster.set_privacy_trampoline(result);
3691        }
3692        #[cfg(not(target_os = "macos"))]
3693        let _ = path;
3694        self
3695    }
3696    /// The flag [`Self::begin_daemon_shutdown`] sets, for the control handler's
3697    /// capability-requirement log to read.
3698    pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
3699        self.child_roster.shutdown_flag()
3700    }
3701
3702    /// The first step of an announced daemon shutdown, before the notice and
3703    /// before any connection is closed.
3704    ///
3705    /// Sets the daemon-shutdown flag first: from here on no module is
3706    /// respawned (crash restart, operator restart, or swap), and every child
3707    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
3708    /// the module exits on the EOF this shutdown gives it or is signalled by a
3709    /// service manager that kills the whole cgroup. Then writes the journal's
3710    /// shutdown marker, which records the instant and closes this daemon
3711    /// incarnation's stretch of the journal.
3712    #[cfg(unix)]
3713    pub(crate) fn begin_daemon_shutdown(&self) {
3714        self.child_roster.close();
3715        if let Some(journal) = &self.terminal_journal {
3716            journal.stamp_shutdown();
3717        }
3718    }
3719
3720    /// Announce a cut while established connections can still carry replies.
3721    /// These budgets bound the attempt to deliver notices and the wait for
3722    /// routes to drain; neither guarantees delivery or child completion, and
3723    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
3724    #[cfg(unix)]
3725    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
3726        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
3727        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
3728        let Some(forwarding) = &self.forwarding else {
3729            return Ok(());
3730        };
3731        let notice_started = Instant::now();
3732        let modules = forwarding
3733            .begin_daemon_drain()
3734            .map_err(SuperviseError::Forwarding)?;
3735        let deadline_ms =
3736            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
3737        let shutdown_reason = self
3738            .supervisor_handle
3739            .as_ref()
3740            .map(SupervisorHandle::take_daemon_shutdown_reason)
3741            .unwrap_or(RouteCloseReason::Disable);
3742        let mut modules_told = 0;
3743        let mut notices = tokio::task::JoinSet::new();
3744        let mut drains = Vec::new();
3745        for connection in forwarding
3746            .module_connections()
3747            .map_err(SuperviseError::Forwarding)?
3748        {
3749            let target = ModuleDrainTarget {
3750                endpoint: connection.endpoint,
3751                sink: connection.sink,
3752                negotiated_ver: connection.negotiated_ver,
3753                abandoned_bindings: Vec::new(),
3754                excluded_subscriptions: 0,
3755            };
3756            notices.spawn(async move {
3757                (
3758                    true,
3759                    send_module_draining(
3760                        &connection.module_id,
3761                        shutdown_reason,
3762                        deadline_ms,
3763                        &target,
3764                        true,
3765                    )
3766                    .await,
3767                )
3768            });
3769        }
3770        for (module_id, endpoint) in modules {
3771            let Some(target) = forwarding
3772                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
3773                .map_err(SuperviseError::Forwarding)?
3774            else {
3775                continue;
3776            };
3777            let routes = forwarding
3778                .endpoint_routes(target.endpoint)
3779                .map_err(SuperviseError::Forwarding)?;
3780            // Only an announced restart promises that modules are coming back.
3781            // Client route notices retain their existing reconnect semantics.
3782            let mut recipients = Vec::new();
3783            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
3784            for route in routes {
3785                let client = route.goodbye_target;
3786                if let Some((_, channels)) = clients
3787                    .iter_mut()
3788                    .find(|(existing, _)| existing.connection_id == client.connection_id)
3789                {
3790                    channels.push(client.channel);
3791                } else {
3792                    let channel = client.channel;
3793                    clients.push((client, vec![channel]));
3794                }
3795            }
3796            for (client, mut channels) in clients {
3797                channels.sort_unstable();
3798                channels.dedup();
3799                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
3800                    module_id: module_id.clone(),
3801                    channels,
3802                    reason: RouteCloseReason::Restart,
3803                })
3804                .expect("route closing serializes");
3805                recipients.push((client.sink, client.negotiated_ver, closing));
3806            }
3807            for (sink, version, body) in recipients {
3808                notices.spawn(async move {
3809                    let frame = Frame::build_with_version(
3810                        version,
3811                        FrameType::Push,
3812                        control_flags(),
3813                        0,
3814                        0,
3815                        0,
3816                        body,
3817                    )
3818                    .expect("bounded lifecycle notice frame builds");
3819                    (false, sink.send_flushed(frame).await.is_ok())
3820                });
3821            }
3822            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3823            drains.push((module_id, target.endpoint, gauges));
3824        }
3825        // A quiet forwarding table is not proof that queued notices reached the
3826        // socket. Wait for writer flush acknowledgements before testing quiescence.
3827        let notice_deadline = Instant::now() + NOTICE_BUDGET;
3828        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3829            match result {
3830                Ok((true, true)) => modules_told += 1,
3831                Ok((false, true)) => {}
3832                result => warn!(?result, "daemon shutdown notice delivery failed"),
3833            }
3834        }
3835        notices.abort_all();
3836        info!(
3837            ?shutdown_reason,
3838            modules_told, "daemon shutdown module notices delivered"
3839        );
3840        let notice_elapsed = notice_started.elapsed();
3841        let drain_started = Instant::now();
3842        let deadline = drain_started + DRAIN_BUDGET;
3843        let mut waits = tokio::task::JoinSet::new();
3844        for (module_id, endpoint, gauges) in drains {
3845            let forwarding = Arc::clone(forwarding);
3846            let mut runtime = self.runtime_config();
3847            runtime.health.cadence = Duration::from_millis(100);
3848            waits.spawn(async move {
3849                let wait_result = wait_for_forwarding_quiescence(
3850                    &forwarding,
3851                    &module_id,
3852                    &runtime,
3853                    endpoint,
3854                    deadline,
3855                    &gauges,
3856                    DrainScope::Endpoint(endpoint),
3857                    &mut DrainGaugeTrace::new(&gauges),
3858                )
3859                .await;
3860                // Store the measurement before the task returns: quiescence
3861                // may be reached just as the deadline stops the join loop from
3862                // collecting results. Unfinished or failed waits report false.
3863                forwarding
3864                    .record_daemon_drain_result(
3865                        endpoint,
3866                        drained_after_quiescence_wait(&wait_result),
3867                    )
3868                    .map_err(SuperviseError::Forwarding)?;
3869                wait_result
3870            });
3871        }
3872        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3873            if !matches!(result, Ok(Ok(true))) {
3874                warn!(?result, "daemon shutdown drain did not reach quiescence");
3875            }
3876        }
3877        // Measure only the forwarding drain, not child teardown or the caller's
3878        // signal delivery and process-reaping delays.
3879        info!(
3880            notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3881            drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3882            "daemon shutdown forwarding drain complete"
3883        );
3884        Ok(())
3885    }
3886
3887    /// The last step of a graceful daemon shutdown, after the notice and the
3888    /// drain: send every registered module a module GOODBYE, the same planned
3889    /// stop signal `ck module stop` gives, then close every connection so each
3890    /// subc module sees EOF and starts its own teardown, then end every
3891    /// supervised child that has not exited
3892    /// by its own deadline (its drain budget, capped). Modules lead their own
3893    /// process groups, so a
3894    /// service manager's group kill no longer reaches them; without this a
3895    /// child that does not stop on EOF (every `protocol: "none"` child, which
3896    /// has no connection) would outlive the daemon. Every wait is bounded (see
3897    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
3898    #[cfg(unix)]
3899    pub(crate) async fn end_children_for_daemon_shutdown(
3900        &self,
3901        already_escalated: bool,
3902        escalate: impl std::future::Future<Output = ()>,
3903    ) {
3904        tokio::pin!(escalate);
3905        let mut escalated = already_escalated;
3906        if let Some(forwarding) = &self.forwarding {
3907            let reason = CloseReason::new(
3908                "daemon_shutdown",
3909                "the daemon is exiting after its shutdown notice and drain",
3910            );
3911            if escalated {
3912                // The operator asked to stop waiting: queue the GOODBYEs but
3913                // do not wait for them to be written.
3914                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3915            } else {
3916                tokio::select! {
3917                    biased;
3918                    _ = escalate.as_mut() => {
3919                        info!("second SIGTERM: abandoning module GOODBYE delivery");
3920                        escalated = true;
3921                    }
3922                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3923                }
3924            }
3925            let closed = forwarding.close_all_connections(&reason);
3926            debug!(closed, "closed established connections for daemon shutdown");
3927        }
3928        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
3929        // already completed and must not be polled again; the child shutdown
3930        // wait is told it is escalated and gets a future that never fires.
3931        let escalated_here = escalated && !already_escalated;
3932        let remaining_escalate = async move {
3933            if escalated_here {
3934                std::future::pending::<()>().await;
3935            } else {
3936                escalate.await;
3937            }
3938        };
3939        crate::child_roster::end_children_for_daemon_shutdown(
3940            &self.child_roster,
3941            escalated,
3942            remaining_escalate,
3943        )
3944        .await;
3945    }
3946
3947    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3948        Self {
3949            registry,
3950            config_delivery: None,
3951            restart_policy,
3952            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3953            connection_file_path: None,
3954            capture_logs_dir: None,
3955            forwarding: None,
3956            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3957            supervisor_handle: None,
3958            health: HealthConfig::default(),
3959            host_health: HostHealth::default(),
3960            daemon_start_clock: crate::clock::StartClock::capture(),
3961            terminal_journal: None,
3962            spawn_events: SpawnEventFeed::default(),
3963            provenance_probe: ExecutableIdentityProbe::default(),
3964            child_roster: ChildRoster::default(),
3965            #[cfg(target_os = "linux")]
3966            cgroup_placement: None,
3967            #[cfg(test)]
3968            test_after_first_spawn: AfterFirstSpawnHook::default(),
3969        }
3970    }
3971
3972    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3973        self.drain_timeout = drain_timeout;
3974        self
3975    }
3976
3977    /// The drain budget a module gets when its config entry sets none.
3978    pub(crate) fn default_drain_timeout(&self) -> Duration {
3979        self.drain_timeout
3980    }
3981
3982    pub fn with_process_liveness(
3983        mut self,
3984        process_liveness: Arc<SupervisorProcessLiveness>,
3985    ) -> Self {
3986        self.process_liveness = process_liveness;
3987        self
3988    }
3989
3990    /// Connect configuration delivery to observed process lifecycle events.
3991    pub fn with_config_runtime(
3992        mut self,
3993        owner: Arc<crate::config_gate::runtime::RuntimeOwner>,
3994    ) -> Self {
3995        if let Some(handle) = &self.supervisor_handle {
3996            handle.install_binding_retirement(&owner);
3997        }
3998        self.config_delivery = Some(config_delivery::ConfigDelivery(owner));
3999        self
4000    }
4001
4002    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
4003        self.connection_file_path = Some(connection_file_path.into());
4004        self
4005    }
4006
4007    /// Enables daemon-owned capture files for supervised stdout and stderr.
4008    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
4009        self.capture_logs_dir = Some(logs_dir.into());
4010        self
4011    }
4012
4013    /// Names this daemon lifetime in spawn events, independently of whether a
4014    /// terminal journal is configured.
4015    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
4016        // A millisecond start stamp can repeat after clock rollback or a rapid
4017        // restart. Use the connection file's random daemon_id instead: it already
4018        // identifies this daemon lifetime independently of the wall clock.
4019        self.spawn_events.configure_incarnation(daemon_incarnation);
4020        self
4021    }
4022
4023    /// Enables best-effort history shared by every supervised module. Without
4024    /// it, terminal history is kept only in each module's in-memory ring.
4025    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
4026        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
4027        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
4028            path,
4029            daemon_incarnation,
4030        )));
4031        this
4032    }
4033
4034    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
4035        self.child_roster.counters = forwarding.counters();
4036        self.forwarding = Some(forwarding);
4037        self
4038    }
4039
4040    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
4041        if let Some(delivery) = &self.config_delivery {
4042            supervisor_handle.install_binding_retirement(&delivery.0);
4043        }
4044        self.host_health = supervisor_handle.host_health.clone();
4045        self.spawn_events = supervisor_handle.spawn_events.clone();
4046        self.supervisor_handle = Some(supervisor_handle);
4047        self
4048    }
4049
4050    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
4051        self.health = health;
4052        self
4053    }
4054
4055    /// Keeps a record of every live child at `path`, rewritten on each spawn and
4056    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
4057    /// record is kept.
4058    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
4059        self.child_roster.record_to(path.into());
4060        self
4061    }
4062
4063    #[cfg(target_os = "linux")]
4064    pub fn with_cgroup_placement(
4065        mut self,
4066        cgroup_placement: Option<subc_cgroup::Placement>,
4067    ) -> Self {
4068        self.child_roster
4069            .set_cgroup_placement(cgroup_placement.clone());
4070        self.cgroup_placement = cgroup_placement;
4071        self
4072    }
4073
4074    /// Spawn `spec.program` and start monitoring it.
4075    ///
4076    /// The child is expected to parse `--subc <connection-file-path>`, read the
4077    /// TCP+key connection file, authenticate to the already-running listener, and
4078    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
4079    ///
4080    /// This synchronous embedder API blocks on launch admission and store I/O.
4081    /// Do not call it on an async runtime thread; use [`Self::spawn_async`].
4082    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4083        validate_spec(&spec)?;
4084        self.establish_identity(&spec);
4085
4086        let runtime = self.runtime_config();
4087        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4088        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4089        let registration_watch = watch_registration(&self.registry, &spec);
4090        let spawned = spawn_child(
4091            &spec,
4092            runtime.connection_file_path.as_deref(),
4093            self.supervisor_handle.as_ref(),
4094            &runtime.stderr_ring,
4095            runtime.capture_logs_dir.as_deref(),
4096            &runtime.child_roster,
4097            runtime.spawn_delivery(),
4098            #[cfg(target_os = "linux")]
4099            runtime.cgroup_placement.as_ref(),
4100            runtime.register_timeout,
4101        );
4102        #[cfg(test)]
4103        self.test_after_first_spawn.run(&spec.module_id);
4104        let mut child = match spawned {
4105            Ok(child) => child,
4106            Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4107                defer_config_spawn(
4108                    &spec,
4109                    &runtime,
4110                    &snapshot,
4111                    RespawnKind::Spawn,
4112                    &runtime.failure_context(runtime.recovery_is_daemon),
4113                )?;
4114                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4115            }
4116            Err(SuperviseError::ConfigHeld { .. }) => {
4117                hold_config_spawn(
4118                    &spec,
4119                    &runtime,
4120                    &snapshot,
4121                    &runtime.failure_context(runtime.recovery_is_daemon),
4122                )?;
4123                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4124            }
4125            Err(err) => {
4126                // Unlike the configured paths, a failed `spawn` leaves nothing
4127                // on the roster, so the module must not stay marked configured.
4128                self.abandon_unrostered(&spec.module_id);
4129                return Err(err);
4130            }
4131        };
4132        arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4133        self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4134        if let Some(spawn) = activity_spawn {
4135            spawn.complete(&snapshot);
4136        }
4137
4138        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4139    }
4140
4141    /// Spawn without running configuration integrity or keychain checks on the
4142    /// runtime thread. Cancellation during preparation cannot create a child.
4143    pub async fn spawn_async(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4144        validate_spec(&spec)?;
4145        self.establish_identity(&spec);
4146        let mut identity = AsyncInitialIdentity {
4147            supervisor: self,
4148            module: spec.module_id.clone(),
4149            committed: false,
4150        };
4151        let runtime = self.runtime_config();
4152        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4153        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4154        let registration_watch = watch_registration(&self.registry, &spec);
4155        let spawned = spawn_child_async(&spec, &runtime).await;
4156        #[cfg(test)]
4157        self.test_after_first_spawn.run(&spec.module_id);
4158        let mut child = match spawned {
4159            Ok(child) => child,
4160            Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4161                defer_config_spawn(
4162                    &spec,
4163                    &runtime,
4164                    &snapshot,
4165                    RespawnKind::Spawn,
4166                    &runtime.failure_context(runtime.recovery_is_daemon),
4167                )?;
4168                identity.committed = true;
4169                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4170            }
4171            Err(SuperviseError::ConfigHeld { .. }) => {
4172                hold_config_spawn(
4173                    &spec,
4174                    &runtime,
4175                    &snapshot,
4176                    &runtime.failure_context(runtime.recovery_is_daemon),
4177                )?;
4178                identity.committed = true;
4179                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4180            }
4181            Err(err) => {
4182                self.abandon_unrostered(&spec.module_id);
4183                return Err(err);
4184            }
4185        };
4186        arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4187        self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4188        if let Some(spawn) = activity_spawn {
4189            spawn.complete(&snapshot);
4190        }
4191        identity.committed = true;
4192        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4193    }
4194
4195    /// Make `spec`'s module count as configured, with its identity gates
4196    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
4197    /// exists.
4198    ///
4199    /// Every path that takes on a new module calls this before `spawn_child`.
4200    /// The order is the point: the child can connect, register, sync its
4201    /// scopes and ask about them as soon as it is spawned, and the module is
4202    /// only put on the roster after `spawn_child` returns. Were the mark set
4203    /// with the roster entry, a fast child would see its own owner reported
4204    /// as not configured, and a scoped `route.open` in that window would be
4205    /// refused as terminal `scope_not_live` ("will never sync") instead of
4206    /// retryable `scope_not_synced`.
4207    fn establish_identity(&self, spec: &ModuleSpec) {
4208        if let Some(supervisor_handle) = &self.supervisor_handle {
4209            supervisor_handle.apply_identity_configuration(spec);
4210            supervisor_handle.mark_configured(&spec.module_id);
4211        }
4212    }
4213
4214    /// Take back [`Self::establish_identity`]'s configured mark when the
4215    /// module will not be put on the roster after all.
4216    fn abandon_unrostered(&self, module_id: &str) {
4217        if let Some(supervisor_handle) = &self.supervisor_handle {
4218            supervisor_handle.unmark_configured_unless_rostered(module_id);
4219        }
4220    }
4221
4222    /// Record a freshly spawned first process as running. On failure the
4223    /// module never reaches the roster, so its configured mark is taken back.
4224    fn mark_first_process_running(
4225        &self,
4226        spec: &ModuleSpec,
4227        runtime: &SupervisorRuntimeConfig,
4228        snapshot: &SharedSnapshot,
4229        child: &mut SupervisedChild,
4230    ) -> Result<(), SuperviseError> {
4231        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
4232            containment::terminate_blocking(&runtime.child_roster.containment);
4233            self.abandon_unrostered(&spec.module_id);
4234            return Err(err);
4235        }
4236        self.process_liveness
4237            .track(spec.module_id.clone(), Arc::clone(snapshot));
4238        Ok(())
4239    }
4240
4241    /// Start supervising a module declared in daemon configuration.
4242    ///
4243    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
4244    /// failures in the supervisor handle so operator-facing `supervisor.list`
4245    /// reflects every configured module while daemon startup continues.
4246    pub fn supervise_configured(
4247        &self,
4248        spec: ModuleSpec,
4249        enabled: bool,
4250    ) -> Result<SupervisedModule, SuperviseError> {
4251        self.supervise_configured_requested(
4252            spec,
4253            enabled,
4254            self.health.clone(),
4255            None,
4256            self.restart_policy,
4257            DEFAULT_REGISTER_TIMEOUT,
4258            None,
4259        )
4260    }
4261
4262    /// Supervise a configured module with its own health, drain, and crash
4263    /// budget. The restart policy is per-module because the config file is:
4264    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
4265    /// module that is expensive to restart should not be forced onto the same
4266    /// budget as one that is cheap.
4267    pub fn supervise_configured_with_health(
4268        &self,
4269        spec: ModuleSpec,
4270        enabled: bool,
4271        health: HealthConfig,
4272        drain_timeout_ms: Option<u64>,
4273        restart_policy: RestartPolicy,
4274    ) -> Result<SupervisedModule, SuperviseError> {
4275        self.supervise_configured_requested(
4276            spec,
4277            enabled,
4278            health,
4279            drain_timeout_ms,
4280            restart_policy,
4281            DEFAULT_REGISTER_TIMEOUT,
4282            None,
4283        )
4284    }
4285
4286    #[allow(clippy::too_many_arguments)]
4287    pub(crate) fn supervise_configured_requested(
4288        &self,
4289        spec: ModuleSpec,
4290        enabled: bool,
4291        health: HealthConfig,
4292        drain_timeout_ms: Option<u64>,
4293        restart_policy: RestartPolicy,
4294        register_timeout: Duration,
4295        requester: Option<Principal>,
4296    ) -> Result<SupervisedModule, SuperviseError> {
4297        let registered = self.register_configured_with_health(
4298            spec,
4299            enabled,
4300            health,
4301            drain_timeout_ms,
4302            restart_policy,
4303            register_timeout,
4304        )?;
4305        let daemon_initiated = requester.is_none();
4306        self.start_registered_configured(registered, requester, daemon_initiated)
4307    }
4308
4309    #[allow(clippy::too_many_arguments)]
4310    pub(crate) async fn supervise_configured_requested_async(
4311        &self,
4312        spec: ModuleSpec,
4313        enabled: bool,
4314        health: HealthConfig,
4315        drain_timeout_ms: Option<u64>,
4316        restart_policy: RestartPolicy,
4317        register_timeout: Duration,
4318        requester: Option<Principal>,
4319    ) -> Result<SupervisedModule, SuperviseError> {
4320        let registered = self.register_configured_with_health(
4321            spec,
4322            enabled,
4323            health,
4324            drain_timeout_ms,
4325            restart_policy,
4326            register_timeout,
4327        )?;
4328        let daemon_initiated = requester.is_none();
4329        self.start_registered_configured_async(registered, requester, daemon_initiated)
4330            .await
4331    }
4332
4333    /// Record a configured module as starting or disabled, without launching a
4334    /// process or running its monitor. Bootstrap registers the entire roster
4335    /// before publishing discovery, then activates it once children can read
4336    /// the connection file passed in their `--subc` argument.
4337    pub(crate) fn register_configured_with_health(
4338        &self,
4339        spec: ModuleSpec,
4340        enabled: bool,
4341        health: HealthConfig,
4342        drain_timeout_ms: Option<u64>,
4343        restart_policy: RestartPolicy,
4344        register_timeout: Duration,
4345    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
4346        validate_spec(&spec)?;
4347        self.establish_identity(&spec);
4348
4349        let mut runtime = self.runtime_config();
4350        runtime.health = health;
4351        runtime.restart_policy = restart_policy;
4352        runtime.register_timeout = register_timeout;
4353        if let Some(ms) = drain_timeout_ms {
4354            runtime.drain_timeout = Duration::from_millis(ms);
4355            *runtime
4356                .effective_drain_timeout
4357                .lock()
4358                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
4359        }
4360        let snapshot = if enabled {
4361            SupervisorSnapshot::starting()
4362        } else {
4363            SupervisorSnapshot::disabled()
4364        };
4365        let snapshot = {
4366            let mut snapshot = snapshot;
4367            if enabled {
4368                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
4369                {
4370                    snapshot.waiting_for_previous_pid = Some(pid);
4371                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
4372                }
4373            }
4374            snapshot
4375        };
4376        let (module, commands) =
4377            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
4378        Ok(RegisteredConfiguredModule {
4379            spec,
4380            runtime,
4381            module,
4382            commands,
4383        })
4384    }
4385
4386    /// Activate a previously registered module. Queued control requests are
4387    /// served by the monitor after the initial configured spawn has settled.
4388    pub(crate) fn start_registered_configured(
4389        &self,
4390        registered: RegisteredConfiguredModule,
4391        requester: Option<Principal>,
4392        daemon_initiated: bool,
4393    ) -> Result<SupervisedModule, SuperviseError> {
4394        let RegisteredConfiguredModule {
4395            spec,
4396            mut runtime,
4397            module,
4398            commands,
4399        } = registered;
4400        runtime.start_requester = requester;
4401        runtime.recovery_is_daemon = daemon_initiated;
4402        let snapshot = &module.inner.snapshot;
4403        let should_spawn = {
4404            let state = lock_snapshot(snapshot)?;
4405            state.enabled && state.waiting_for_previous_pid.is_none()
4406        };
4407        let child = if should_spawn {
4408            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4409            let registration_watch = watch_registration(&self.registry, &spec);
4410            let spawned = spawn_child(
4411                &spec,
4412                runtime.connection_file_path.as_deref(),
4413                self.supervisor_handle.as_ref(),
4414                &runtime.stderr_ring,
4415                runtime.capture_logs_dir.as_deref(),
4416                &runtime.child_roster,
4417                runtime.spawn_delivery(),
4418                #[cfg(target_os = "linux")]
4419                runtime.cgroup_placement.as_ref(),
4420                runtime.register_timeout,
4421            );
4422            #[cfg(test)]
4423            self.test_after_first_spawn.run(&spec.module_id);
4424            match spawned {
4425                Ok(mut child) => {
4426                    arm_registration_deadline(
4427                        &mut child,
4428                        runtime.register_timeout,
4429                        registration_watch,
4430                    );
4431                    self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4432                    if let Some(spawn) = activity_spawn {
4433                        spawn.complete(snapshot);
4434                    }
4435                    Some(child)
4436                }
4437                Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4438                    defer_config_spawn(
4439                        &spec,
4440                        &runtime,
4441                        snapshot,
4442                        RespawnKind::Spawn,
4443                        &runtime.failure_context(daemon_initiated),
4444                    )?;
4445                    None
4446                }
4447                Err(mut err) => {
4448                    if matches!(err, SuperviseError::ConfigHeld { .. }) {
4449                        match hold_config_spawn(
4450                            &spec,
4451                            &runtime,
4452                            snapshot,
4453                            &runtime.failure_context(daemon_initiated),
4454                        ) {
4455                            Ok(()) => {
4456                                self.start_monitor(spec, runtime, &module, None, commands);
4457                                return Ok(module);
4458                            }
4459                            Err(parking_error) => err = parking_error,
4460                        }
4461                    }
4462                    if runtime.health.critical {
4463                        error!(
4464                            module_id = %spec.module_id,
4465                            program = %spec.program.display(),
4466                            error = %err,
4467                            "critical configured module failed to spawn; marking failed and alerting"
4468                        );
4469                    } else {
4470                        error!(
4471                            module_id = %spec.module_id,
4472                            program = %spec.program.display(),
4473                            error = %err,
4474                            "configured module failed to spawn; marking failed and continuing"
4475                        );
4476                    }
4477                    fail_snapshot(
4478                        snapshot,
4479                        Some(&spec.module_id),
4480                        None,
4481                        &runtime.failure_context(daemon_initiated),
4482                    );
4483                    None
4484                }
4485            }
4486        } else {
4487            None
4488        };
4489        self.start_monitor(spec, runtime, &module, child, commands);
4490        Ok(module)
4491    }
4492
4493    pub(crate) async fn start_registered_configured_async(
4494        &self,
4495        registered: RegisteredConfiguredModule,
4496        requester: Option<Principal>,
4497        daemon_initiated: bool,
4498    ) -> Result<SupervisedModule, SuperviseError> {
4499        self.start_registered_with_failure_policy(registered, requester, daemon_initiated, None)
4500            .await
4501    }
4502
4503    pub(crate) async fn start_registered_approved_async(
4504        &self,
4505        registered: RegisteredConfiguredModule,
4506        requester: Principal,
4507        generation: crate::config_gate::runtime::ApprovalIdentity,
4508    ) -> Result<SupervisedModule, SuperviseError> {
4509        self.start_registered_with_failure_policy(
4510            registered,
4511            Some(requester),
4512            false,
4513            Some(generation),
4514        )
4515        .await
4516    }
4517
4518    async fn start_registered_with_failure_policy(
4519        &self,
4520        registered: RegisteredConfiguredModule,
4521        requester: Option<Principal>,
4522        daemon_initiated: bool,
4523        approved_generation: Option<crate::config_gate::runtime::ApprovalIdentity>,
4524    ) -> Result<SupervisedModule, SuperviseError> {
4525        let RegisteredConfiguredModule {
4526            spec,
4527            mut runtime,
4528            module,
4529            commands,
4530        } = registered;
4531        runtime.start_requester = requester;
4532        runtime.recovery_is_daemon = daemon_initiated;
4533        let snapshot = &module.inner.snapshot;
4534        let mut startup_error = None;
4535        let failure = FailureContext {
4536            approval: approved_generation.clone(),
4537            ..runtime.failure_context(daemon_initiated)
4538        };
4539        let should_spawn = {
4540            let state = lock_snapshot(snapshot)?;
4541            state.enabled && state.waiting_for_previous_pid.is_none()
4542        };
4543        let child = if should_spawn {
4544            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4545            let registration_watch = watch_registration(&self.registry, &spec);
4546            let spawned = spawn_child_async(&spec, &runtime).await;
4547            #[cfg(test)]
4548            self.test_after_first_spawn.run(&spec.module_id);
4549            match spawned {
4550                Ok(mut child) => {
4551                    arm_registration_deadline(
4552                        &mut child,
4553                        runtime.register_timeout,
4554                        registration_watch,
4555                    );
4556                    self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4557                    if let Some(spawn) = activity_spawn {
4558                        spawn.complete(snapshot);
4559                    }
4560                    Some(child)
4561                }
4562                Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4563                    defer_config_spawn(&spec, &runtime, snapshot, RespawnKind::Spawn, &failure)?;
4564                    None
4565                }
4566                Err(mut err) => {
4567                    if matches!(err, SuperviseError::ConfigHeld { .. }) {
4568                        match hold_config_spawn(&spec, &runtime, snapshot, &failure) {
4569                            Ok(()) => {
4570                                self.start_monitor(spec, runtime, &module, None, commands);
4571                                return Ok(module);
4572                            }
4573                            Err(parking_error) => err = parking_error,
4574                        }
4575                    }
4576                    if runtime.health.critical {
4577                        error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4578                            "critical configured module failed to spawn; marking failed and alerting");
4579                    } else {
4580                        error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4581                            "configured module failed to spawn; marking failed and continuing");
4582                    }
4583                    fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
4584                    if let Some(generation) = approved_generation {
4585                        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4586                            state.approved_start_failure = Some((generation, err.to_string()));
4587                        });
4588                        startup_error = Some(err);
4589                    }
4590                    None
4591                }
4592            }
4593        } else {
4594            None
4595        };
4596        self.start_monitor(spec, runtime, &module, child, commands);
4597        // Applying an operator-approved configuration must report a spawn error
4598        // without removing the failed module or its monitor. Bootstrap instead
4599        // tolerates failed starts so other configured modules can still run.
4600        startup_error.map_or(Ok(module), Err)
4601    }
4602
4603    fn runtime_config(&self) -> SupervisorRuntimeConfig {
4604        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
4605        SupervisorRuntimeConfig {
4606            registry: self.registry.clone(),
4607            config_delivery: self.config_delivery.clone(),
4608            recovery_is_daemon: true,
4609            start_requester: None,
4610            pending_activity_restart: false,
4611            provenance_probe: self.provenance_probe.clone(),
4612            scheduled_respawn: Arc::default(),
4613            deferred_reload_reply: Arc::default(),
4614            restart_policy: self.restart_policy,
4615            register_timeout: DEFAULT_REGISTER_TIMEOUT,
4616            drain_timeout: self.drain_timeout,
4617            // Shared with this module's roster copy: daemon shutdown waits on
4618            // each child for the module's own drain budget, as resolved now.
4619            child_roster: self
4620                .child_roster
4621                .for_module(Arc::clone(&effective_drain_timeout)),
4622            effective_drain_timeout,
4623            default_drain_timeout: self.drain_timeout,
4624            health: self.health.clone(),
4625            host_health: self.host_health.clone(),
4626            connection_file_path: self.connection_file_path.clone(),
4627            capture_logs_dir: self.capture_logs_dir.clone(),
4628            forwarding: self.forwarding.clone(),
4629            supervisor_handle: self.supervisor_handle.clone(),
4630            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
4631            terminal_ring: Arc::new(Mutex::new(
4632                TerminalRing::new(
4633                    TerminalRingConfig::default(),
4634                    self.daemon_start_clock.started_at_ms(),
4635                )
4636                .with_start_clock(self.daemon_start_clock)
4637                .with_journal(self.terminal_journal.clone())
4638                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
4639            )),
4640            spawn_events: self.spawn_events.clone(),
4641            #[cfg(target_os = "linux")]
4642            cgroup_placement: self.cgroup_placement.clone(),
4643            #[cfg(test)]
4644            test_seed_stale_facts_before_enable_spawn: false,
4645            #[cfg(test)]
4646            test_reload_exit_record_gate: None,
4647        }
4648    }
4649
4650    fn supervised_module(
4651        &self,
4652        spec: ModuleSpec,
4653        runtime: SupervisorRuntimeConfig,
4654        snapshot: SharedSnapshot,
4655        child: Option<SupervisedChild>,
4656    ) -> SupervisedModule {
4657        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
4658        self.start_monitor(spec, runtime, &module, child, commands);
4659        module
4660    }
4661
4662    fn register_module(
4663        &self,
4664        spec: &ModuleSpec,
4665        runtime: &SupervisorRuntimeConfig,
4666        snapshot: SharedSnapshot,
4667    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
4668        runtime.host_health.track(spec, &snapshot);
4669        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
4670            spec: spec.clone(),
4671            enabled: snapshot.lock().unwrap_or_else(|p| p.into_inner()).enabled,
4672            health: runtime.health.clone(),
4673            register_timeout: runtime.register_timeout,
4674            restart_policy: runtime.restart_policy,
4675        }));
4676        let stderr_ring = Arc::clone(&runtime.stderr_ring);
4677        let terminal_ring = Arc::clone(&runtime.terminal_ring);
4678        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
4679        let (tx, rx) = mpsc::channel(4);
4680        let module_id = spec.module_id.clone();
4681        let module = SupervisedModule {
4682            inner: Arc::new(SupervisedModuleInner {
4683                module_id: module_id.clone(),
4684                registry: Arc::clone(&self.registry),
4685                snapshot,
4686                configuration,
4687                stderr_ring,
4688                terminal_ring,
4689                commands: tx,
4690                monitor: Mutex::new(None),
4691                effective_drain_timeout,
4692                provenance_probe: self.provenance_probe.clone(),
4693                retained: runtime.child_roster.retained_view(),
4694            }),
4695        };
4696        // Identity gates and the configured mark must precede the roster entry,
4697        // so a client never sees a module without its admission policy.
4698        if let Some(supervisor_handle) = &self.supervisor_handle {
4699            supervisor_handle.insert(module.clone());
4700        }
4701        (module, rx)
4702    }
4703
4704    fn start_monitor(
4705        &self,
4706        spec: ModuleSpec,
4707        runtime: SupervisorRuntimeConfig,
4708        module: &SupervisedModule,
4709        child: Option<SupervisedChild>,
4710        commands: mpsc::Receiver<SupervisorCommand>,
4711    ) {
4712        let monitor = tokio::spawn(supervise_loop(
4713            spec,
4714            runtime,
4715            Arc::clone(&self.registry),
4716            Arc::clone(&self.process_liveness),
4717            Arc::clone(&module.inner.snapshot),
4718            child,
4719            commands,
4720        ));
4721        *module
4722            .inner
4723            .monitor
4724            .lock()
4725            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
4726    }
4727}
4728
4729impl Default for Supervisor {
4730    fn default() -> Self {
4731        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
4732    }
4733}
4734
4735/// Handle to one supervised child process.
4736#[derive(Clone)]
4737pub struct SupervisedModule {
4738    inner: Arc<SupervisedModuleInner>,
4739}
4740
4741struct SupervisedModuleInner {
4742    retained: crate::child_roster::RetainedView,
4743    module_id: String,
4744    registry: Arc<Registry>,
4745    snapshot: SharedSnapshot,
4746    configuration: Arc<Mutex<SupervisedConfiguration>>,
4747    stderr_ring: Arc<Mutex<StderrRing>>,
4748    terminal_ring: Arc<Mutex<TerminalRing>>,
4749    commands: mpsc::Sender<SupervisorCommand>,
4750    monitor: Mutex<Option<JoinHandle<()>>>,
4751    effective_drain_timeout: Arc<Mutex<Duration>>,
4752    provenance_probe: ExecutableIdentityProbe,
4753}
4754
4755impl fmt::Debug for SupervisedModule {
4756    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4757        f.debug_struct("SupervisedModule")
4758            .field("module_id", &self.inner.module_id)
4759            .field("status", &self.status())
4760            .finish_non_exhaustive()
4761    }
4762}
4763
4764impl SupervisedModule {
4765    pub(crate) fn check_approved_start_failure(
4766        &self,
4767        generation: &crate::config_gate::runtime::ApprovalIdentity,
4768    ) -> Result<bool, SuperviseError> {
4769        let state = lock_snapshot(&self.inner.snapshot)?;
4770        if let Some((failed_generation, detail)) = &state.approved_start_failure {
4771            if failed_generation == generation {
4772                return Err(SuperviseError::ReloadFailed {
4773                    module_id: self.inner.module_id.clone(),
4774                    reason: detail.clone(),
4775                });
4776            }
4777        }
4778        Ok(state.approved_start_failure.is_some())
4779    }
4780
4781    pub(crate) fn has_approved_start_failure(
4782        &self,
4783        identity: &crate::config_gate::runtime::ApprovalIdentity,
4784    ) -> bool {
4785        lock_snapshot(&self.inner.snapshot).is_ok_and(|state| {
4786            state
4787                .approved_start_failure
4788                .as_ref()
4789                .is_some_and(|(failed, _)| failed == identity)
4790        })
4791    }
4792
4793    pub(crate) fn needs_approved_gate_restart(
4794        &self,
4795        approval: &crate::config_gate::runtime::ApprovalIdentity,
4796    ) -> bool {
4797        lock_snapshot(&self.inner.snapshot)
4798            .is_ok_and(|state| state.needs_approved_gate_restart(approval))
4799    }
4800
4801    #[cfg(all(test, unix))]
4802    pub(crate) fn config_hold_origin_for_test(
4803        &self,
4804    ) -> (bool, Option<crate::config_gate::runtime::ApprovalIdentity>) {
4805        let state = lock_snapshot(&self.inner.snapshot).unwrap();
4806        (
4807            state.config_hold_daemon_initiated,
4808            state.config_hold_approval.clone(),
4809        )
4810    }
4811
4812    pub(crate) async fn config_policy_probe(
4813        &self,
4814        connection: crate::ConnectionId,
4815        generation: u64,
4816        attempt: u64,
4817    ) -> Result<(), SuperviseError> {
4818        self.inner
4819            .commands
4820            .send(SupervisorCommand::ConfigPolicyProbe {
4821                connection,
4822                generation,
4823                attempt,
4824            })
4825            .await
4826            .map_err(|_| SuperviseError::CommandClosed {
4827                module_id: self.inner.module_id.clone(),
4828            })
4829    }
4830    pub(crate) async fn config_exit_hold(&self) -> Result<(), SuperviseError> {
4831        let (reply, result) = oneshot::channel();
4832        self.inner
4833            .commands
4834            .send(SupervisorCommand::ConfigExitHold { reply })
4835            .await
4836            .map_err(|_| SuperviseError::CommandClosed {
4837                module_id: self.module_id().to_string(),
4838            })?;
4839        result.await.map_err(|_| SuperviseError::CommandClosed {
4840            module_id: self.module_id().to_string(),
4841        })?
4842    }
4843    pub(crate) async fn config_delivery_failed(
4844        &self,
4845        connection: crate::ConnectionId,
4846        hold: bool,
4847    ) -> Result<(), SuperviseError> {
4848        self.inner
4849            .commands
4850            .send(SupervisorCommand::ConfigDeliveryFailed { connection, hold })
4851            .await
4852            .map_err(|_| SuperviseError::CommandClosed {
4853                module_id: self.module_id().to_string(),
4854            })
4855    }
4856
4857    pub(crate) async fn config_apply_restart(&self, generation: u64) -> Result<(), SuperviseError> {
4858        self.inner
4859            .commands
4860            .send(SupervisorCommand::ConfigApplyRestart {
4861                generation,
4862                supersedes_swap: false,
4863            })
4864            .await
4865            .map_err(|_| SuperviseError::CommandClosed {
4866                module_id: self.inner.module_id.clone(),
4867            })
4868    }
4869    pub fn module_id(&self) -> &str {
4870        &self.inner.module_id
4871    }
4872
4873    /// Test-only: put one probe miss on the streak, the way
4874    /// `handle_health_probe_failure` does, so tests can assert what a later
4875    /// event does to the streak without driving the whole probe loop.
4876    #[cfg(test)]
4877    pub(crate) fn record_health_probe_failure_for_test(
4878        &self,
4879        detail: &str,
4880    ) -> Result<(), SuperviseError> {
4881        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
4882            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
4883            state.health.detail = Some(detail.to_string());
4884        })
4885    }
4886
4887    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
4888        Ok(lock_snapshot(&self.inner.snapshot)?.state)
4889    }
4890
4891    pub(crate) fn registration_end_reason(
4892        &self,
4893    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
4894        let snapshot = lock_snapshot(&self.inner.snapshot)?;
4895        Ok(match snapshot.state {
4896            ModuleState::Draining if snapshot.draining_to_replace => {
4897                Some(RegistrationEndReason::SupervisorRestart)
4898            }
4899            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
4900            ModuleState::Restarting if snapshot.draining_to_replace => {
4901                Some(RegistrationEndReason::SupervisorRestart)
4902            }
4903            _ => None,
4904        })
4905    }
4906
4907    /// The module's retained stderr, newest lines last.
4908    ///
4909    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
4910    /// module, `supervisor.list` renders every module, and putting it in the
4911    /// shared snapshot would make each status read carry a payload almost nobody
4912    /// asked for. Callers that want the text ask for it.
4913    pub fn stderr_tail(
4914        &self,
4915        max_lines: Option<usize>,
4916        max_bytes: Option<usize>,
4917    ) -> StderrTailSnapshot {
4918        self.inner
4919            .stderr_ring
4920            .lock()
4921            .unwrap_or_else(|poisoned| poisoned.into_inner())
4922            .snapshot(max_lines, max_bytes)
4923    }
4924
4925    /// The module's bounded terminal history, oldest retained exit first.
4926    ///
4927    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
4928    /// daemon whose in-memory history was necessarily reset.
4929    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
4930        self.inner
4931            .terminal_ring
4932            .lock()
4933            .unwrap_or_else(|poisoned| poisoned.into_inner())
4934            .snapshot()
4935    }
4936
4937    /// Retained observations from the current ring and all journal generations.
4938    ///
4939    /// Blocking: this reads the journal files. Async callers use
4940    /// `Self::read_durable_terminal_history`.
4941    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
4942        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
4943    }
4944
4945    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
4946    /// read (up to every retained generation) never occupies a runtime worker.
4947    /// Fails only if the blocking task could not finish (runtime shutdown or a
4948    /// panic in the read).
4949    pub(crate) async fn read_durable_terminal_history(
4950        &self,
4951    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
4952        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
4953        let module_id = self.inner.module_id.clone();
4954        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
4955            .await
4956    }
4957
4958    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
4959        self.status_with_snapshot_lock(&self.inner.snapshot, None)
4960            .map(|(status, _)| status)
4961    }
4962
4963    pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
4964        let state = lock_snapshot(&self.inner.snapshot)?;
4965        Ok(state
4966            .failed_retry
4967            .pending
4968            .filter(|_| state.state == ModuleState::Failed && state.enabled)
4969            .map(|retry| {
4970                unix_ms_now().saturating_add(
4971                    u64::try_from(
4972                        retry
4973                            .deadline
4974                            .saturating_duration_since(Instant::now())
4975                            .as_millis(),
4976                    )
4977                    .unwrap_or(u64::MAX),
4978                )
4979            }))
4980    }
4981
4982    #[cfg(all(test, unix))]
4983    pub(crate) fn retry_deadline_for_test(&self) -> Option<Instant> {
4984        lock_snapshot(&self.inner.snapshot)
4985            .unwrap()
4986            .failed_retry
4987            .pending
4988            .map(|retry| retry.deadline)
4989    }
4990
4991    pub(crate) fn record_deliberate_severance(
4992        &self,
4993        identity: ProcessIdentity,
4994    ) -> Result<bool, SuperviseError> {
4995        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4996        if snapshot.pid != Some(identity.pid)
4997            || snapshot.process_start_time != Some(identity.start_time)
4998        {
4999            return Ok(false);
5000        }
5001        snapshot.deliberate_severance = Some(identity);
5002        Ok(true)
5003    }
5004
5005    /// Read status for a channel-0 renderer and report a contended snapshot lock.
5006    ///
5007    /// Internal supervision callers use [`Self::status`] so writer-side machinery
5008    /// does not produce reader-observability logs.
5009    pub(crate) fn status_for_control(
5010        &self,
5011        caller: &'static str,
5012    ) -> Result<ModuleStatus, SuperviseError> {
5013        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
5014            .map(|(status, _)| status)
5015    }
5016
5017    fn status_with_snapshot_lock(
5018        &self,
5019        snapshot: &SharedSnapshot,
5020        caller: Option<&'static str>,
5021    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
5022        let restart_policy = self
5023            .inner
5024            .configuration
5025            .lock()
5026            .map_err(|_| SuperviseError::StatePoisoned {
5027                module_id: Some(self.inner.module_id.clone()),
5028            })?
5029            .restart_policy;
5030        let mut guard = match caller {
5031            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
5032            None => lock_snapshot(snapshot)?,
5033        };
5034        // Read the budget through the pruning path so a reader sees the same
5035        // in-window count the restart decision would use, not a stale total.
5036        let restart_count = guard.crash_restarts_in_window(restart_policy.window, Instant::now());
5037        let snapshot = guard.clone();
5038        drop(guard);
5039        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
5040            SuperviseError::StatePoisoned {
5041                module_id: Some(self.inner.module_id.clone()),
5042            }
5043        })?;
5044        let registration_active = self
5045            .inner
5046            .registry
5047            .get_module(&self.inner.module_id)
5048            .map_err(SuperviseError::Registry)?
5049            .is_some();
5050        let protocol = snapshot
5051            .spawned_protocol
5052            .unwrap_or(self.declared_protocol()?);
5053        let running_process =
5054            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
5055        // Registration is the difference between the two protocols and the only
5056        // one: a subc module that has not registered cannot serve a request even
5057        // though its process is up, and a `none` module never registers at all,
5058        // so requiring it there would pin `live` to false for the whole life of
5059        // a perfectly healthy process.
5060        let live = match protocol {
5061            ModuleProtocol::Subc => running_process && registration_active,
5062            ModuleProtocol::None => running_process,
5063        };
5064        // Read only the retained cache: a status read must not resend signals,
5065        // rewrite the record, or keep the containment resource alive.
5066        let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
5067        let carry_configured = self
5068            .inner
5069            .configuration
5070            .lock()
5071            .map_err(|_| SuperviseError::StatePoisoned {
5072                module_id: Some(self.inner.module_id.clone()),
5073            })?
5074            .spec
5075            .carry_containment;
5076        let recovery_block = carries_containment(
5077            carry_configured,
5078            self.inner
5079                .retained
5080                .has_carried_container(&self.inner.module_id),
5081        )
5082        .then(|| self.inner.retained.recovery_block())
5083        .flatten();
5084        let launch_nonce_source = observed_launch_nonce_source(&snapshot);
5085        let pid = snapshot.reported_pid();
5086        let mut health = snapshot.health;
5087        if let Some(detail) = recovery_block
5088            .as_ref()
5089            .or_else(|| retained.as_ref().map(|(_, detail)| detail))
5090        {
5091            health.detail = Some(match health.detail {
5092                Some(previous) => format!("{previous}; {detail}"),
5093                None => detail.clone(),
5094            });
5095        }
5096
5097        Ok((
5098            ModuleStatus {
5099                admission_refused: snapshot.admission_refused,
5100                legacy_registration_spawn: snapshot.legacy_registration_spawn,
5101                registration_declaration: snapshot.registration_declaration.clone(),
5102                stop_reason: snapshot.stop_reason,
5103                waiting_for_previous_pid: retained
5104                    .as_ref()
5105                    .map(|(pid, _)| *pid)
5106                    .filter(|pid| *pid > 0)
5107                    .or(snapshot.waiting_for_previous_pid),
5108                waiting_for_previous_detail: recovery_block
5109                    .or_else(|| retained.map(|(_, detail)| detail))
5110                    .or(snapshot.waiting_for_previous_detail),
5111                module_id: self.inner.module_id.clone(),
5112                launch_nonce_source,
5113                state: snapshot.state,
5114                enabled: snapshot.enabled,
5115                process_alive: snapshot.process_alive,
5116                registration_active,
5117                protocol,
5118                live,
5119                restart_count,
5120                lifetime_restarts: snapshot.lifetime_restarts,
5121                spawn_generation: snapshot.spawn_generation,
5122                swap_outcome: snapshot.swap_outcome,
5123                max_restarts: restart_policy.max_restarts,
5124                restart_window: restart_policy.window,
5125                drain_timeout,
5126                restart_backoff: restart_policy.backoff,
5127                restart_max_backoff: restart_policy.max_backoff,
5128                pid,
5129                spawned_at_ms: snapshot.spawned_at_ms,
5130                spawned_from: snapshot.spawned_from,
5131                launch_spec_digest: snapshot.launch_spec_digest,
5132                process_start_time: snapshot.process_start_time,
5133                last_exit: snapshot.last_exit,
5134                health,
5135            },
5136            snapshot.spawned_file_identity.clone(),
5137        ))
5138    }
5139
5140    #[cfg(test)]
5141    pub(crate) fn hold_snapshot_for_test(
5142        &self,
5143        acquired: std::sync::mpsc::Sender<()>,
5144        hold: Duration,
5145    ) -> std::thread::JoinHandle<()> {
5146        let snapshot = Arc::clone(&self.inner.snapshot);
5147        std::thread::spawn(move || {
5148            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
5149            acquired
5150                .send(())
5151                .expect("test receiver waits for snapshot lock");
5152            std::thread::sleep(hold);
5153        })
5154    }
5155
5156    /// The status and the running-image check for `supervisor.provenance`,
5157    /// taken from one status read. The exec acknowledgement can land between
5158    /// two separate reads, and the reply would then pair "no pid yet" with an
5159    /// image observed after the module started, which describes no single
5160    /// moment.
5161    pub(crate) async fn status_and_running_image_agreement(
5162        &self,
5163    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
5164        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
5165        let image = self
5166            .inner
5167            .provenance_probe
5168            .observe(
5169                status.pid,
5170                status.spawned_from.as_deref(),
5171                identity,
5172                status.process_start_time,
5173            )
5174            .await;
5175        Ok((status, image))
5176    }
5177
5178    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
5179        let snapshot = match lock_snapshot(&self.inner.snapshot) {
5180            Ok(snapshot) => snapshot.clone(),
5181            Err(_) => {
5182                return subc_control::RunningImageAgreement::Unavailable {
5183                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
5184                };
5185            }
5186        };
5187        self.inner
5188            .provenance_probe
5189            .observe(
5190                snapshot.reported_pid(),
5191                snapshot.spawned_from.as_deref(),
5192                snapshot.spawned_file_identity.clone(),
5193                snapshot.process_start_time,
5194            )
5195            .await
5196    }
5197
5198    /// Memory and CPU time of the module's current process, read now. Only the
5199    /// process the supervisor spawned is read, not processes it has started.
5200    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
5201        #[cfg(windows)]
5202        {
5203            let image = lock_snapshot(&self.inner.snapshot)
5204                .ok()
5205                .and_then(|snapshot| {
5206                    snapshot
5207                        .spawned_file_identity
5208                        .as_ref()
5209                        .and_then(|identity| identity.windows_image.clone())
5210                });
5211            match image {
5212                Some(image) => {
5213                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
5214                }
5215                None => subc_control::ChildResourceUsage::Unavailable {
5216                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
5217                },
5218            }
5219        }
5220        #[cfg(not(windows))]
5221        {
5222            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
5223                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
5224                Err(_) => {
5225                    return subc_control::ChildResourceUsage::Unavailable {
5226                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
5227                    }
5228                }
5229            };
5230            crate::child_resources::read(pid, start_time)
5231        }
5232    }
5233
5234    #[cfg(all(test, any(target_os = "linux", windows)))]
5235    pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
5236        self.inner
5237            .snapshot
5238            .try_lock()
5239            .map(|snapshot| snapshot.subtree_target.clone())
5240            .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
5241    }
5242
5243    #[cfg(test)]
5244    pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
5245        self.inner.snapshot.lock().unwrap().subtree_target = target;
5246    }
5247
5248    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
5249        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
5250        Ok(match snapshot.state {
5251            ModuleState::Restarting => true,
5252            ModuleState::Failed | ModuleState::Disabled => false,
5253            _ if snapshot.enabled && snapshot.respawn_pending => true,
5254            _ => daemon_will_restart(
5255                &mut snapshot,
5256                &self
5257                    .inner
5258                    .configuration
5259                    .lock()
5260                    .map_err(|_| SuperviseError::StatePoisoned {
5261                        module_id: Some(self.inner.module_id.clone()),
5262                    })?
5263                    .restart_policy,
5264                Instant::now(),
5265            ),
5266        })
5267    }
5268
5269    #[cfg(test)]
5270    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
5271        self.is_warming_with_snapshot_lock(None)
5272    }
5273
5274    pub(crate) fn is_warming_for_control(
5275        &self,
5276        caller: &'static str,
5277    ) -> Result<bool, SuperviseError> {
5278        self.is_warming_with_snapshot_lock(Some(caller))
5279    }
5280
5281    fn is_warming_with_snapshot_lock(
5282        &self,
5283        caller: Option<&'static str>,
5284    ) -> Result<bool, SuperviseError> {
5285        let snapshot = match caller {
5286            Some(caller) => {
5287                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
5288            }
5289            None => lock_snapshot(&self.inner.snapshot)?,
5290        }
5291        .clone();
5292        Ok(matches!(
5293            snapshot.state,
5294            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
5295        ))
5296    }
5297
5298    /// Drain the module and stop monitoring it.
5299    pub async fn drain(&self) -> Result<(), SuperviseError> {
5300        self.stop().await
5301    }
5302
5303    pub(crate) async fn retire_requested(
5304        &self,
5305        requester: Option<Principal>,
5306    ) -> Result<(), SuperviseError> {
5307        match self.state()? {
5308            ModuleState::Stopped => return Ok(()),
5309            ModuleState::Failed
5310                if lock_snapshot(&self.inner.snapshot)?
5311                    .failed_retry
5312                    .pending
5313                    .is_none() =>
5314            {
5315                return Ok(())
5316            }
5317            ModuleState::Failed
5318            | ModuleState::Starting
5319            | ModuleState::Running
5320            | ModuleState::Unresponsive
5321            | ModuleState::Restarting
5322            | ModuleState::Draining
5323            | ModuleState::Disabled => {}
5324        }
5325
5326        let (reply_tx, reply_rx) = oneshot::channel();
5327        self.inner
5328            .commands
5329            .send(SupervisorCommand::Retire {
5330                requester,
5331                reply: reply_tx,
5332            })
5333            .await
5334            .map_err(|_| SuperviseError::CommandClosed {
5335                module_id: self.inner.module_id.clone(),
5336            })?;
5337        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5338            module_id: self.inner.module_id.clone(),
5339        })?
5340    }
5341
5342    pub async fn stop(&self) -> Result<(), SuperviseError> {
5343        self.stop_requested(None).await
5344    }
5345
5346    pub(crate) async fn stop_requested(
5347        &self,
5348        requester: Option<Principal>,
5349    ) -> Result<(), SuperviseError> {
5350        match self.state()? {
5351            ModuleState::Stopped => return Ok(()),
5352            ModuleState::Failed
5353                if lock_snapshot(&self.inner.snapshot)?
5354                    .failed_retry
5355                    .pending
5356                    .is_none() =>
5357            {
5358                return Ok(())
5359            }
5360            ModuleState::Failed
5361            | ModuleState::Starting
5362            | ModuleState::Running
5363            | ModuleState::Unresponsive
5364            | ModuleState::Restarting
5365            | ModuleState::Draining
5366            | ModuleState::Disabled => {}
5367        }
5368
5369        let (reply_tx, reply_rx) = oneshot::channel();
5370        self.inner
5371            .commands
5372            .send(SupervisorCommand::Drain {
5373                requester,
5374                reply: reply_tx,
5375            })
5376            .await
5377            .map_err(|_| SuperviseError::CommandClosed {
5378                module_id: self.inner.module_id.clone(),
5379            })?;
5380        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5381            module_id: self.inner.module_id.clone(),
5382        })?
5383    }
5384
5385    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
5386        self.restart_requested(drain_timeout_ms, None).await
5387    }
5388
5389    pub(crate) async fn restart_requested(
5390        &self,
5391        drain_timeout_ms: Option<u64>,
5392        requester: Option<Principal>,
5393    ) -> Result<(), SuperviseError> {
5394        let queued_at = Instant::now();
5395        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
5396        let (reply_tx, reply_rx) = oneshot::channel();
5397        self.inner
5398            .commands
5399            .send(SupervisorCommand::Restart {
5400                requester,
5401                drain_timeout_ms,
5402                received_at_generation,
5403                queued_at,
5404                reply: reply_tx,
5405            })
5406            .await
5407            .map_err(|_| SuperviseError::CommandClosed {
5408                module_id: self.inner.module_id.clone(),
5409            })?;
5410        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5411            module_id: self.inner.module_id.clone(),
5412        })?
5413    }
5414
5415    /// Blue/green restart: see `SupervisorCommand::Swap` and the
5416    /// `supervisor_swap` module. Returns once the swap has cut over (the old
5417    /// process then drains in the background of the supervise loop), has been
5418    /// superseded by an approved configuration restart, or has failed. Both
5419    /// supersession and failure leave the old process serving until replacement.
5420    pub async fn swap(
5421        &self,
5422        ready_timeout: Option<Duration>,
5423    ) -> Result<SwapOutcome, SuperviseError> {
5424        self.swap_requested(ready_timeout, None).await
5425    }
5426
5427    pub(crate) async fn swap_requested(
5428        &self,
5429        ready_timeout: Option<Duration>,
5430        requester: Option<Principal>,
5431    ) -> Result<SwapOutcome, SuperviseError> {
5432        let (reply_tx, reply_rx) = oneshot::channel();
5433        self.inner
5434            .commands
5435            .send(SupervisorCommand::Swap {
5436                requester,
5437                ready_timeout,
5438                reply: reply_tx,
5439            })
5440            .await
5441            .map_err(|_| SuperviseError::CommandClosed {
5442                module_id: self.inner.module_id.clone(),
5443            })?;
5444        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5445            module_id: self.inner.module_id.clone(),
5446        })?
5447    }
5448
5449    pub async fn reload(&self) -> Result<(), SuperviseError> {
5450        self.reload_requested(None).await
5451    }
5452
5453    pub(crate) async fn reload_requested(
5454        &self,
5455        requester: Option<Principal>,
5456    ) -> Result<(), SuperviseError> {
5457        let (reply_tx, reply_rx) = oneshot::channel();
5458        self.inner
5459            .commands
5460            .send(SupervisorCommand::Reload {
5461                requester,
5462                reply: reply_tx,
5463            })
5464            .await
5465            .map_err(|_| SuperviseError::CommandClosed {
5466                module_id: self.inner.module_id.clone(),
5467            })?;
5468        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5469            module_id: self.inner.module_id.clone(),
5470        })?
5471    }
5472
5473    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
5474        self.set_enabled_requested(enabled, None).await
5475    }
5476
5477    pub(crate) async fn set_enabled_requested(
5478        &self,
5479        enabled: bool,
5480        requester: Option<Principal>,
5481    ) -> Result<bool, SuperviseError> {
5482        self.set_enabled_with_budget_policy(enabled, requester, true)
5483            .await
5484    }
5485
5486    pub(crate) async fn release_config_hold(&self) -> Result<bool, SuperviseError> {
5487        self.set_enabled_with_budget_policy(true, None, false).await
5488    }
5489
5490    #[cfg(all(test, unix))]
5491    pub(crate) fn seed_crash_budget_for_test(&self, count: u32) {
5492        let policy = self.inner.configuration.lock().unwrap().restart_policy;
5493        let mut state = self.inner.snapshot.lock().unwrap();
5494        for _ in 0..count {
5495            state.record_crash_restart(&policy, Instant::now());
5496        }
5497    }
5498
5499    async fn set_enabled_with_budget_policy(
5500        &self,
5501        enabled: bool,
5502        requester: Option<Principal>,
5503        reset_budget: bool,
5504    ) -> Result<bool, SuperviseError> {
5505        let (reply_tx, reply_rx) = oneshot::channel();
5506        self.inner
5507            .commands
5508            .send(SupervisorCommand::SetEnabled {
5509                requester,
5510                enabled,
5511                reset_budget,
5512                reply: reply_tx,
5513            })
5514            .await
5515            .map_err(|_| SuperviseError::CommandClosed {
5516                module_id: self.inner.module_id.clone(),
5517            })?;
5518        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5519            module_id: self.inner.module_id.clone(),
5520        })?
5521    }
5522
5523    /// The current process's protocol, or the configured protocol when down.
5524    /// A rescan stores the next launch spec without changing how an existing
5525    /// process registers, serves routes, is probed, or exits.
5526    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
5527        let configured = self
5528            .inner
5529            .configuration
5530            .lock()
5531            .map_err(|_| SuperviseError::StatePoisoned {
5532                module_id: Some(self.inner.module_id.clone()),
5533            })?
5534            .spec
5535            .protocol;
5536        let state = lock_snapshot(&self.inner.snapshot)?;
5537        Ok(state.spawned_protocol.unwrap_or(configured))
5538    }
5539
5540    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
5541        let configuration =
5542            self.inner
5543                .configuration
5544                .lock()
5545                .map_err(|_| SuperviseError::StatePoisoned {
5546                    module_id: Some(self.inner.module_id.clone()),
5547                })?;
5548        Ok((configuration.spec.clone(), configuration.health.clone()))
5549    }
5550
5551    pub(crate) fn configured_register_timeout(&self) -> Result<Duration, SuperviseError> {
5552        self.inner
5553            .configuration
5554            .lock()
5555            .map(|configuration| configuration.register_timeout)
5556            .map_err(|_| SuperviseError::StatePoisoned {
5557                module_id: Some(self.inner.module_id.clone()),
5558            })
5559    }
5560
5561    pub(crate) fn configured_enabled(&self) -> Result<bool, SuperviseError> {
5562        self.inner
5563            .configuration
5564            .lock()
5565            .map(|configuration| configuration.enabled)
5566            .map_err(|_| SuperviseError::StatePoisoned {
5567                module_id: Some(self.inner.module_id.clone()),
5568            })
5569    }
5570
5571    /// Replace this module's launch spec, keeping its health and drain policy,
5572    /// the way a rescan does for a changed config entry. The running process is
5573    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
5574    #[cfg(any(test, feature = "test-support"))]
5575    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
5576        let (_, health) = self.configuration()?;
5577        let drain_timeout_ms = u64::try_from(
5578            self.inner
5579                .effective_drain_timeout
5580                .lock()
5581                .unwrap_or_else(|poisoned| poisoned.into_inner())
5582                .as_millis(),
5583        )
5584        .ok();
5585        self.update_configuration(spec, health, drain_timeout_ms)
5586            .await
5587    }
5588
5589    pub(crate) async fn reconcile_approved_configuration(
5590        &self,
5591        approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5592    ) -> Result<(), SuperviseError> {
5593        let entry = approved
5594            .module_entry(&self.inner.module_id)
5595            .map_err(|error| SuperviseError::InvalidSpec {
5596                reason: error.to_string(),
5597            })?;
5598        validate_spec(&entry.module_spec())?;
5599        let (reply, received) = oneshot::channel();
5600        self.inner
5601            .commands
5602            .send(SupervisorCommand::ApprovedConfiguration { approved, reply })
5603            .await
5604            .map_err(|_| SuperviseError::CommandClosed {
5605                module_id: self.inner.module_id.clone(),
5606            })?;
5607        received.await.map_err(|_| SuperviseError::CommandClosed {
5608            module_id: self.inner.module_id.clone(),
5609        })??;
5610        let mut configuration =
5611            self.inner
5612                .configuration
5613                .lock()
5614                .map_err(|_| SuperviseError::StatePoisoned {
5615                    module_id: Some(self.inner.module_id.clone()),
5616                })?;
5617        configuration.spec = entry.module_spec();
5618        configuration.enabled = entry.enabled;
5619        configuration.health = entry.health;
5620        configuration.register_timeout = entry.register_timeout;
5621        configuration.restart_policy = entry.restart;
5622        Ok(())
5623    }
5624
5625    pub(crate) async fn update_configuration(
5626        &self,
5627        spec: ModuleSpec,
5628        health: HealthConfig,
5629        drain_timeout_ms: Option<u64>,
5630    ) -> Result<(), SuperviseError> {
5631        if spec.module_id != self.inner.module_id {
5632            return Err(SuperviseError::InvalidSpec {
5633                reason: "a supervised module's module_id cannot be changed".to_string(),
5634            });
5635        }
5636        validate_spec(&spec)?;
5637        let (reply_tx, reply_rx) = oneshot::channel();
5638        self.inner
5639            .commands
5640            .send(SupervisorCommand::UpdateConfiguration {
5641                spec: spec.clone(),
5642                health: health.clone(),
5643                drain_timeout_ms,
5644                reply: reply_tx,
5645            })
5646            .await
5647            .map_err(|_| SuperviseError::CommandClosed {
5648                module_id: self.inner.module_id.clone(),
5649            })?;
5650        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5651            module_id: self.inner.module_id.clone(),
5652        })??;
5653        let mut configuration =
5654            self.inner
5655                .configuration
5656                .lock()
5657                .map_err(|_| SuperviseError::StatePoisoned {
5658                    module_id: Some(self.inner.module_id.clone()),
5659                })?;
5660        configuration.spec = spec;
5661        configuration.health = health;
5662        Ok(())
5663    }
5664}
5665
5666impl Drop for SupervisedModuleInner {
5667    fn drop(&mut self) {
5668        let Ok(mut monitor) = self.monitor.lock() else {
5669            return;
5670        };
5671        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
5672            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
5673                state.state = ModuleState::Stopped;
5674                clear_current_process_facts(state);
5675            });
5676            monitor.abort();
5677        }
5678        let _ = monitor.take();
5679    }
5680}
5681
5682#[derive(Debug)]
5683enum SupervisorCommand {
5684    ApprovedConfiguration {
5685        approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5686        reply: oneshot::Sender<Result<(), SuperviseError>>,
5687    },
5688    ConfigPolicyProbe {
5689        connection: crate::ConnectionId,
5690        generation: u64,
5691        attempt: u64,
5692    },
5693    ConfigApplyRestart {
5694        generation: u64,
5695        supersedes_swap: bool,
5696    },
5697    ConfigExitHold {
5698        reply: oneshot::Sender<Result<(), SuperviseError>>,
5699    },
5700    ConfigDeliveryFailed {
5701        connection: crate::ConnectionId,
5702        hold: bool,
5703    },
5704    Drain {
5705        requester: Option<Principal>,
5706        reply: oneshot::Sender<Result<(), SuperviseError>>,
5707    },
5708    Retire {
5709        requester: Option<Principal>,
5710        reply: oneshot::Sender<Result<(), SuperviseError>>,
5711    },
5712    Restart {
5713        requester: Option<Principal>,
5714        /// Operator override for this one restart's drain budget, in ms. `None`
5715        /// uses the module's configured/default budget; `Some(0)` cuts
5716        /// immediately (wedge bounce: a stuck request never settles, so
5717        /// waiting only delays recovery).
5718        drain_timeout_ms: Option<u64>,
5719        /// The module's `spawn_generation` when the request was received, before
5720        /// it waited in the command queue. A queued restart whose module has
5721        /// since spawned a newer process is already satisfied (see the handler).
5722        received_at_generation: u64,
5723        /// When the request entered the command queue, so the handler can log
5724        /// how long it waited behind the loop's other work.
5725        queued_at: Instant,
5726        reply: oneshot::Sender<Result<(), SuperviseError>>,
5727    },
5728    Reload {
5729        requester: Option<Principal>,
5730        reply: oneshot::Sender<Result<(), SuperviseError>>,
5731    },
5732    SetEnabled {
5733        reset_budget: bool,
5734        requester: Option<Principal>,
5735        enabled: bool,
5736        reply: oneshot::Sender<Result<bool, SuperviseError>>,
5737    },
5738    UpdateConfiguration {
5739        spec: ModuleSpec,
5740        health: HealthConfig,
5741        /// Per-module drain override from the new config; `None` re-resolves to
5742        /// the supervisor-wide default.
5743        drain_timeout_ms: Option<u64>,
5744        reply: oneshot::Sender<Result<(), SuperviseError>>,
5745    },
5746    Swap {
5747        requester: Option<Principal>,
5748        /// How long the candidate may take to register and declare itself
5749        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
5750        ready_timeout: Option<Duration>,
5751        /// Answered at cutover or failure; the incumbent's drain follows.
5752        reply: oneshot::Sender<Result<SwapOutcome, SuperviseError>>,
5753    },
5754}
5755
5756/// Successful daemon-local swap disposition. Supersession leaves the incumbent
5757/// serving until an approved replacement starts with the latest configured
5758/// command; the legacy control acknowledgment does not carry this disposition.
5759#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5760pub enum SwapOutcome {
5761    Cutover,
5762    SupersededByConfigRestart,
5763}
5764
5765impl SwapOutcome {
5766    pub fn as_str(self) -> &'static str {
5767        match self {
5768            Self::Cutover => "cutover",
5769            Self::SupersededByConfigRestart => "superseded_by_config_restart",
5770        }
5771    }
5772}
5773
5774#[derive(Debug)]
5775pub enum SuperviseError {
5776    ConfigHeld {
5777        module_id: String,
5778        code: String,
5779        detail: String,
5780    },
5781    ConfigDeliveryDeferred {
5782        module_id: String,
5783    },
5784    InvalidSpec {
5785        reason: String,
5786    },
5787    Spawn {
5788        program: PathBuf,
5789        source: io::Error,
5790        cgroup_path: Option<PathBuf>,
5791    },
5792    Cgroup {
5793        module_id: String,
5794        source: io::Error,
5795    },
5796    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
5797    /// than spawn a reserved module without its identity binding.
5798    LaunchNonce {
5799        reason: String,
5800    },
5801    Wait {
5802        module_id: String,
5803        source: io::Error,
5804    },
5805    Kill {
5806        module_id: String,
5807        source: io::Error,
5808    },
5809    Forwarding(ForwardingError),
5810    Registry(RegistryError),
5811    ReloadUnavailable {
5812        module_id: String,
5813        reason: String,
5814    },
5815    /// An operator restart/reload was requested for a module that is currently
5816    /// disabled. Restart/reload cycle a *running* module; a disabled module must
5817    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
5818    /// by a restart, so these commands are rejected instead of re-enabling it.
5819    Disabled {
5820        module_id: String,
5821    },
5822    ReloadFailed {
5823        module_id: String,
5824        reason: String,
5825    },
5826    RegistrationStillActive {
5827        module_id: String,
5828        waited: Duration,
5829    },
5830    StatePoisoned {
5831        module_id: Option<String>,
5832    },
5833    CommandClosed {
5834        module_id: String,
5835    },
5836    /// A restart or reload arrived while a swap's candidate was warming. The
5837    /// swap owns the module until it cuts over or fails; a stop or disable
5838    /// would have aborted it instead.
5839    SwapInProgress {
5840        module_id: String,
5841    },
5842    /// A swap was refused before anything was spawned.
5843    SwapRefused {
5844        module_id: String,
5845        reason: SwapRefusal,
5846    },
5847    /// A swap spawned a candidate and gave up on it. The candidate has been
5848    /// killed and its slot freed; the incumbent was left serving and was never
5849    /// drained, except in the one `CutoverLost` case described on that arm.
5850    SwapFailed {
5851        module_id: String,
5852        arm: SwapFailureArm,
5853        detail: String,
5854        /// How the candidate exited, when it exited on its own before the
5855        /// supervisor gave up on it.
5856        candidate_exit: Option<ExitReport>,
5857    },
5858}
5859
5860/// Why a swap was refused before a candidate was spawned.
5861#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5862pub enum SwapRefusal {
5863    /// The module's config does not declare `overlap: "safe"`.
5864    OverlapExclusive,
5865    /// The module is not registered, so there is no incumbent to keep serving
5866    /// and nothing a swap would improve on; a plain restart is the tool.
5867    NotRegistered,
5868    /// The module does not speak the subc wire, so a candidate could never
5869    /// register or declare itself ready.
5870    ProtocolNone,
5871    /// The supervisor lacks the forwarding table (to cut routes over) or the
5872    /// shared handle (to admit the candidate's HELLO) that a swap needs.
5873    NotConfigured,
5874    /// A swap is already open for this module.
5875    AlreadySwapping,
5876}
5877
5878impl SwapRefusal {
5879    pub fn as_str(self) -> &'static str {
5880        match self {
5881            Self::OverlapExclusive => "overlap_exclusive",
5882            Self::NotRegistered => "not_registered",
5883            Self::ProtocolNone => "protocol_none",
5884            Self::NotConfigured => "not_configured",
5885            Self::AlreadySwapping => "already_swapping",
5886        }
5887    }
5888}
5889
5890/// Which failure arm ended a swap. Every arm but one leaves the incumbent
5891/// serving and undrained; see `CutoverLost`.
5892#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5893pub enum SwapFailureArm {
5894    /// The candidate process could not be started.
5895    SpawnFailed,
5896    /// The candidate did not register within the readiness budget.
5897    NeverRegistered,
5898    /// The candidate registered but did not declare itself ready in time.
5899    NeverReady,
5900    /// The candidate exited before cutover.
5901    CandidateExited,
5902    /// The candidate declared itself ready but failed its health probe.
5903    CandidateUnhealthy,
5904    /// An operator stop, disable or retire arrived while the candidate warmed.
5905    /// The candidate was killed and the operator's command then carried out on
5906    /// the incumbent.
5907    Interrupted,
5908    /// The candidate's connection closed at the moment of cutover. If it
5909    /// closed before forwarding moved, the incumbent is untouched. If it closed
5910    /// between the forwarding and registry halves of cutover, forwarding can no
5911    /// longer route to the incumbent, so the module is restarted plainly.
5912    CutoverLost,
5913}
5914
5915impl SwapFailureArm {
5916    pub fn as_str(self) -> &'static str {
5917        match self {
5918            Self::SpawnFailed => "spawn_failed",
5919            Self::NeverRegistered => "never_registered",
5920            Self::NeverReady => "never_ready",
5921            Self::CandidateExited => "candidate_exited",
5922            Self::CandidateUnhealthy => "candidate_unhealthy",
5923            Self::Interrupted => "interrupted",
5924            Self::CutoverLost => "cutover_lost",
5925        }
5926    }
5927}
5928
5929impl SuperviseError {
5930    pub(crate) fn is_config_gate_error(&self) -> bool {
5931        match self {
5932            Self::ConfigHeld { .. } => true,
5933            Self::Spawn { source, .. } => source
5934                .get_ref()
5935                .is_some_and(|cause| cause.is::<crate::config_gate::GateError>()),
5936            _ => false,
5937        }
5938    }
5939}
5940
5941impl fmt::Display for SuperviseError {
5942    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5943        match self {
5944            Self::ConfigHeld { module_id, code, detail } => write!(f, "{code}: module '{module_id}' held: {detail}"),
5945            Self::ConfigDeliveryDeferred { module_id } => write!(f, "config_gated: module '{module_id}' is waiting for its exited delivery target's failure record"),
5946            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
5947            Self::Spawn {
5948                program,
5949                source,
5950                cgroup_path: Some(cgroup_path),
5951            } => write!(
5952                f,
5953                "failed to place module in cgroup '{}' while spawning '{}': {source}",
5954                cgroup_path.display(),
5955                program.display()
5956            ),
5957            Self::Spawn {
5958                program,
5959                source,
5960                cgroup_path: None,
5961            } => write!(
5962                f,
5963                "failed to spawn module '{}': {source}",
5964                program.display()
5965            ),
5966            Self::Cgroup { module_id, source } => {
5967                write!(
5968                    f,
5969                    "failed to prepare cgroup for module '{module_id}': {source}"
5970                )
5971            }
5972            Self::LaunchNonce { reason } => {
5973                write!(
5974                    f,
5975                    "failed to generate reserved-module launch nonce: {reason}"
5976                )
5977            }
5978            Self::Wait { module_id, source } => {
5979                write!(f, "failed to wait for module '{module_id}': {source}")
5980            }
5981            Self::Kill { module_id, source } => {
5982                write!(f, "failed to kill module '{module_id}': {source}")
5983            }
5984            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
5985            Self::Registry(err) => write!(f, "registry error: {err}"),
5986            Self::ReloadUnavailable { module_id, reason } => {
5987                write!(f, "reload unavailable for module '{module_id}': {reason}")
5988            }
5989            Self::Disabled { module_id } => {
5990                write!(
5991                    f,
5992                    "module '{module_id}' is disabled; enable it before restart or reload"
5993                )
5994            }
5995            Self::ReloadFailed { module_id, reason } => {
5996                write!(f, "reload failed for module '{module_id}': {reason}")
5997            }
5998            Self::RegistrationStillActive { module_id, waited } => write!(
5999                f,
6000                "module '{module_id}' registration remained active after waiting {waited:?}"
6001            ),
6002            Self::StatePoisoned { module_id } => match module_id {
6003                Some(module_id) => {
6004                    write!(f, "supervisor state for module '{module_id}' was poisoned")
6005                }
6006                None => write!(f, "supervisor state was poisoned"),
6007            },
6008            Self::CommandClosed { module_id } => {
6009                write!(
6010                    f,
6011                    "supervisor command channel for module '{module_id}' is closed"
6012                )
6013            }
6014            Self::SwapInProgress { module_id } => write!(
6015                f,
6016                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
6017            ),
6018            Self::SwapRefused { module_id, reason } => match reason {
6019                SwapRefusal::OverlapExclusive => write!(
6020                    f,
6021                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
6022                ),
6023                SwapRefusal::NotRegistered => write!(
6024                    f,
6025                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
6026                ),
6027                SwapRefusal::ProtocolNone => write!(
6028                    f,
6029                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
6030                ),
6031                SwapRefusal::NotConfigured => write!(
6032                    f,
6033                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
6034                ),
6035                SwapRefusal::AlreadySwapping => {
6036                    write!(f, "module '{module_id}' is already being swapped")
6037                }
6038            },
6039            Self::SwapFailed {
6040                module_id,
6041                arm,
6042                detail,
6043                ..
6044            } => write!(
6045                f,
6046                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
6047                arm.as_str()
6048            ),
6049        }
6050    }
6051}
6052
6053impl Error for SuperviseError {
6054    fn source(&self) -> Option<&(dyn Error + 'static)> {
6055        match self {
6056            Self::Spawn { source, .. }
6057            | Self::Cgroup { source, .. }
6058            | Self::Wait { source, .. }
6059            | Self::Kill { source, .. } => Some(source),
6060            Self::Forwarding(err) => Some(err),
6061            Self::Registry(err) => Some(err),
6062            Self::LaunchNonce { .. }
6063            | Self::ConfigDeliveryDeferred { .. }
6064            | Self::ConfigHeld { .. }
6065            | Self::InvalidSpec { .. }
6066            | Self::ReloadUnavailable { .. }
6067            | Self::Disabled { .. }
6068            | Self::ReloadFailed { .. }
6069            | Self::RegistrationStillActive { .. }
6070            | Self::StatePoisoned { .. }
6071            | Self::CommandClosed { .. }
6072            | Self::SwapInProgress { .. }
6073            | Self::SwapRefused { .. }
6074            | Self::SwapFailed { .. } => None,
6075        }
6076    }
6077}
6078
6079pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
6080    if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
6081        return Err(SuperviseError::InvalidSpec {
6082            reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
6083        });
6084    }
6085    if spec.module_id.trim().is_empty() {
6086        return Err(SuperviseError::InvalidSpec {
6087            reason: "module_id must not be empty".to_string(),
6088        });
6089    }
6090
6091    Ok(())
6092}
6093
6094#[derive(Debug, Default)]
6095struct HealthProbeRuntime {
6096    spawn_generation: Option<u64>,
6097    configured_health: Option<HealthConfig>,
6098    registered_connection: Option<crate::ConnectionId>,
6099    advertised: bool,
6100    next_probe_at: Option<Instant>,
6101    probe_index: u64,
6102}
6103
6104fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
6105    lock_snapshot(snapshot)
6106        .ok()
6107        .and_then(|state| state.spawned_protocol)
6108        .unwrap_or(spec.protocol)
6109}
6110
6111impl HealthProbeRuntime {
6112    fn refresh_registration(
6113        &mut self,
6114        spec: &ModuleSpec,
6115        runtime: &SupervisorRuntimeConfig,
6116        registry: &Registry,
6117        snapshot: &SharedSnapshot,
6118    ) {
6119        let generation = lock_snapshot(snapshot)
6120            .ok()
6121            .map(|state| state.spawn_generation);
6122        if self.spawn_generation != generation {
6123            // A replacement cannot reuse a deadline that is already due for its
6124            // predecessor. HTTP children get a full cadence to start; wire
6125            // children still wait for their own health-advertising HELLO.
6126            *self = Self {
6127                spawn_generation: generation,
6128                ..Self::default()
6129            };
6130        }
6131        if self.configured_health.as_ref() != Some(&runtime.health) {
6132            self.configured_health = Some(runtime.health.clone());
6133            self.next_probe_at = None;
6134            self.registered_connection = None;
6135            self.probe_index = 0;
6136        }
6137        // A non-wire process never registers. Only an explicitly configured
6138        // HTTP endpoint can arm its health probe; an absent HELLO is not a
6139        // health failure for that kind of process.
6140        if running_protocol(spec, snapshot) == ModuleProtocol::None {
6141            self.registered_connection = None;
6142            self.advertised = runtime.health.http.is_some();
6143            if !self.advertised {
6144                self.next_probe_at = None;
6145                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6146                    let unknown = ModuleHealthStatus::default();
6147                    if state.health != unknown {
6148                        state.health = unknown;
6149                        state.last_probe_evidence = None;
6150                    }
6151                });
6152            } else if self.next_probe_at.is_none() {
6153                self.next_probe_at = Some(
6154                    Instant::now()
6155                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6156                );
6157            }
6158            return;
6159        }
6160
6161        let registration = match registry.get_module(&spec.module_id) {
6162            Ok(registration) => registration,
6163            Err(err) => {
6164                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
6165                self.advertised = false;
6166                self.next_probe_at = None;
6167                return;
6168            }
6169        };
6170
6171        let Some(registration) = registration else {
6172            self.registered_connection = None;
6173            self.advertised = false;
6174            self.next_probe_at = None;
6175            return;
6176        };
6177
6178        let advertised = registration
6179            .control_ops
6180            .iter()
6181            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
6182        if !advertised {
6183            self.registered_connection = Some(registration.connection_id);
6184            self.advertised = false;
6185            self.next_probe_at = None;
6186            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6187                if state.health.status != SupervisorHealthStatus::Unknown
6188                    || state.health.consecutive_failures != 0
6189                    || state.health.last_probe_ms.is_some()
6190                    || state.health.detail.is_some()
6191                    || state.health.metrics.is_some()
6192                {
6193                    state.health.status = SupervisorHealthStatus::Unknown;
6194                    state.last_probe_evidence = None;
6195                    state.health.consecutive_failures = 0;
6196                    state.health.last_probe_ms = None;
6197                    state.health.detail = None;
6198                    state.health.metrics = None;
6199                }
6200            });
6201            return;
6202        }
6203
6204        let reregistered = self.registered_connection != Some(registration.connection_id);
6205        self.registered_connection = Some(registration.connection_id);
6206        self.advertised = true;
6207        if reregistered || self.next_probe_at.is_none() {
6208            self.probe_index = 0;
6209            self.next_probe_at = Some(
6210                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6211            );
6212            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6213                state.health.status = SupervisorHealthStatus::Unknown;
6214                state.last_probe_evidence = None;
6215                state.health.consecutive_failures = 0;
6216                state.health.detail = None;
6217                state.health.metrics = None;
6218            });
6219        }
6220    }
6221
6222    fn wake_at(&self) -> Option<Instant> {
6223        if !self.advertised {
6224            return None;
6225        }
6226        self.next_probe_at
6227    }
6228
6229    fn due(&self) -> bool {
6230        self.advertised
6231            && self
6232                .next_probe_at
6233                .is_some_and(|next| Instant::now() >= next)
6234    }
6235
6236    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
6237        self.probe_index = self.probe_index.wrapping_add(1);
6238        self.next_probe_at = Some(
6239            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
6240        );
6241    }
6242}
6243
6244/// Evidence is separate from diagnostic prose so host starvation can hold only
6245/// silence-based restarts. A missing lane is still actionable during overload;
6246/// a timeout can instead mean the host has not scheduled a healthy module.
6247#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6248enum HealthProbeEvidence {
6249    /// The module's control lane is gone. Proof of death.
6250    LaneDead,
6251    /// No reply within the deadline. Proves nothing about the module's state.
6252    NoAnswer,
6253    /// The module replied, but not with a usable health report. Proves it is alive.
6254    BadAnswer,
6255    /// The daemon could not ask. Says nothing about the module.
6256    Misconfigured,
6257}
6258
6259#[derive(Debug)]
6260struct HealthProbeError {
6261    evidence: HealthProbeEvidence,
6262    message: String,
6263}
6264
6265impl HealthProbeError {
6266    fn lane_dead(message: impl Into<String>) -> Self {
6267        Self::with(HealthProbeEvidence::LaneDead, message)
6268    }
6269
6270    fn no_answer(message: impl Into<String>) -> Self {
6271        Self::with(HealthProbeEvidence::NoAnswer, message)
6272    }
6273
6274    fn bad_answer(message: impl Into<String>) -> Self {
6275        Self::with(HealthProbeEvidence::BadAnswer, message)
6276    }
6277
6278    fn misconfigured(message: impl Into<String>) -> Self {
6279        Self::with(HealthProbeEvidence::Misconfigured, message)
6280    }
6281
6282    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
6283        Self {
6284            evidence,
6285            message: message.into(),
6286        }
6287    }
6288
6289    /// Whether this observation is proof the module cannot serve.
6290    ///
6291    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
6292    /// variant that fires under CPU starvation, and treating it as proof is the
6293    /// defect this enum exists to make impossible to reintroduce silently.
6294    ///
6295    /// Outside a host-overload hold, silence still reaches the normal failure
6296    /// threshold so an isolated, genuinely wedged module can recover.
6297    #[allow(dead_code)]
6298    fn is_proof_of_death(&self) -> bool {
6299        matches!(self.evidence, HealthProbeEvidence::LaneDead)
6300    }
6301
6302    /// Short stable label for logs and the health snapshot.
6303    ///
6304    /// An operator reading `ck health` currently cannot tell "the module is gone"
6305    /// from "the module did not answer in five seconds", because both render as
6306    /// prose in the same field. These labels are what make the two
6307    /// distinguishable at a glance, and they are what a later restart-policy
6308    /// change will be argued from.
6309    fn label(&self) -> &'static str {
6310        match self.evidence {
6311            HealthProbeEvidence::LaneDead => "lane-dead",
6312            HealthProbeEvidence::NoAnswer => "no-answer",
6313            HealthProbeEvidence::BadAnswer => "bad-answer",
6314            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
6315        }
6316    }
6317}
6318
6319impl fmt::Display for HealthProbeError {
6320    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
6321        f.write_str(&self.message)
6322    }
6323}
6324
6325async fn run_health_probe_cycle(
6326    spec: &ModuleSpec,
6327    runtime: &SupervisorRuntimeConfig,
6328    registry: &Registry,
6329    process_liveness: &SupervisorProcessLiveness,
6330    snapshot: &SharedSnapshot,
6331    child: &mut Option<SupervisedChild>,
6332) {
6333    let now_ms = unix_ms_now();
6334    let delivery_probe = match &runtime.config_delivery {
6335        Some(delivery) => {
6336            delivery
6337                .health_probe(&spec.module_id, runtime.forwarding.as_deref(), registry)
6338                .await
6339        }
6340        None => None,
6341    };
6342    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
6343        .then_some(runtime.health.http.as_deref())
6344        .flatten();
6345    let result = match http {
6346        Some(url) => probe_http_health(url, runtime.health.deadline)
6347            .await
6348            .map(|report| (report, None)),
6349        None => probe_module_health_from(&spec.module_id, runtime, None)
6350            .await
6351            .map(|(report, probed)| (report, Some(probed))),
6352    };
6353    match result {
6354        Ok((report, probed)) => {
6355            handle_health_report(
6356                spec,
6357                runtime,
6358                registry,
6359                process_liveness,
6360                snapshot,
6361                child,
6362                report,
6363                now_ms,
6364                delivery_probe,
6365                probed,
6366            )
6367            .await;
6368        }
6369        Err(err) => {
6370            if http.is_some() {
6371                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6372                    state.health.status = SupervisorHealthStatus::Failing;
6373                });
6374            }
6375            handle_health_probe_failure(
6376                spec,
6377                runtime,
6378                registry,
6379                process_liveness,
6380                snapshot,
6381                child,
6382                err,
6383                now_ms,
6384                delivery_probe,
6385            )
6386            .await;
6387        }
6388    }
6389}
6390
6391pub(crate) struct HttpProbeTarget<'a> {
6392    address: std::net::SocketAddr,
6393    localhost: bool,
6394    authority: &'a str,
6395    path: String,
6396}
6397
6398/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
6399/// or TLS. A URL cannot turn a local health check into an outbound connection.
6400pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
6401    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
6402        return Err("must not contain whitespace, controls, or a fragment".into());
6403    }
6404    let rest = url
6405        .strip_prefix("http://")
6406        .ok_or("must use plain http://")?;
6407    let split = rest.find(['/', '?']).unwrap_or(rest.len());
6408    let (authority, suffix) = rest.split_at(split);
6409    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
6410        ("::1", rest)
6411    } else {
6412        let split = authority.find(':').unwrap_or(authority.len());
6413        authority.split_at(split)
6414    };
6415    let ip: std::net::IpAddr = match host {
6416        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
6417        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
6418        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
6419    };
6420    let port = if port.is_empty() {
6421        80
6422    } else {
6423        port.strip_prefix(':')
6424            .and_then(|p| p.parse::<u16>().ok())
6425            .filter(|p| *p > 0)
6426            .ok_or("must have a valid nonzero TCP port")?
6427    };
6428    let path = if suffix.is_empty() {
6429        "/".into()
6430    } else if suffix.starts_with('?') {
6431        format!("/{suffix}")
6432    } else {
6433        suffix.into()
6434    };
6435    Ok(HttpProbeTarget {
6436        address: std::net::SocketAddr::new(ip, port),
6437        localhost: host == "localhost",
6438        authority,
6439        path,
6440    })
6441}
6442
6443async fn probe_http_health(
6444    url: &str,
6445    deadline: Duration,
6446) -> Result<HealthReport, HealthProbeError> {
6447    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
6448    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
6449    // Keep partial diagnostics outside the timed future so cancellation does
6450    // not discard a status line or body bytes already received.
6451    let mut response_status = String::new();
6452    let mut body = Vec::new();
6453    let probe = async {
6454        // Resolve localhost ourselves so a hosts-file override cannot turn
6455        // this into an outbound request, while IPv6-only local servers work.
6456        let connection = match tokio::net::TcpStream::connect(target.address).await {
6457            Err(_) if target.localhost => {
6458                tokio::net::TcpStream::connect((
6459                    std::net::Ipv6Addr::LOCALHOST,
6460                    target.address.port(),
6461                ))
6462                .await
6463            }
6464            result => result,
6465        };
6466        let mut stream = connection.map_err(|error| {
6467            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
6468        })?;
6469        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
6470            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
6471        let mut reader = BufReader::new(stream);
6472        let mut budget = 16 * 1024;
6473        let status = http_line(&mut reader, &mut budget).await?;
6474        let mut words = status.split_ascii_whitespace();
6475        let version = words.next();
6476        let code = words
6477            .next()
6478            .filter(|word| word.len() == 3)
6479            .and_then(|word| word.parse::<u16>().ok());
6480        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
6481            || !code.is_some_and(|code| (100..600).contains(&code))
6482        {
6483            return Err(HealthProbeError::bad_answer(format!(
6484                "invalid HTTP status: {status}"
6485            )));
6486        }
6487        let code = code.expect("validated status code");
6488        response_status = status.clone();
6489        let mut length = None;
6490        let mut chunked = false;
6491        loop {
6492            let line = http_line(&mut reader, &mut budget).await?;
6493            if line.is_empty() {
6494                break;
6495            }
6496            if let Some((name, value)) = line.split_once(':') {
6497                if name.eq_ignore_ascii_case("content-length") {
6498                    length = Some(value.trim().parse::<u64>().map_err(|_| {
6499                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
6500                    })?);
6501                } else if name.eq_ignore_ascii_case("transfer-encoding") {
6502                    chunked = value.trim().eq_ignore_ascii_case("chunked");
6503                }
6504            }
6505        }
6506        if chunked {
6507            while body.len() < 200 {
6508                let line = http_line(&mut reader, &mut budget).await?;
6509                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
6510                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
6511                if size == 0 {
6512                    break;
6513                }
6514                let count = size.min((200 - body.len()) as u64) as usize;
6515                let start = body.len();
6516                (&mut reader)
6517                    .take(count as u64)
6518                    .read_to_end(&mut body)
6519                    .await
6520                    .map_err(|error| {
6521                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6522                    })?;
6523                if body.len() - start != count {
6524                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
6525                }
6526                if size > count as u64 || body.len() == 200 {
6527                    break;
6528                }
6529                if !http_line(&mut reader, &mut budget).await?.is_empty() {
6530                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
6531                }
6532            }
6533        } else {
6534            reader
6535                .take(length.unwrap_or(200).min(200))
6536                .read_to_end(&mut body)
6537                .await
6538                .map_err(|error| {
6539                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6540                })?;
6541        }
6542        if (200..300).contains(&code) {
6543            Ok(HealthReport::ok())
6544        } else {
6545            Err(HealthProbeError::bad_answer(
6546                "HTTP health endpoint returned non-2xx",
6547            ))
6548        }
6549    };
6550    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
6551        Err(HealthProbeError::no_answer(format!(
6552            "HTTP probe timed out after {deadline:?}"
6553        )))
6554    });
6555    if let Err(error) = &mut result {
6556        if !response_status.is_empty() {
6557            error.message = format!(
6558                "{}; {response_status}: {}",
6559                error.message,
6560                String::from_utf8_lossy(&body)
6561            );
6562        }
6563    }
6564    result
6565}
6566
6567async fn http_line(
6568    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
6569    remaining: &mut usize,
6570) -> Result<String, HealthProbeError> {
6571    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
6572    let mut line = Vec::new();
6573    (&mut *reader)
6574        .take(*remaining as u64)
6575        .read_until(b'\n', &mut line)
6576        .await
6577        .map_err(|error| {
6578            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
6579        })?;
6580    *remaining -= line.len();
6581    if !line.ends_with(b"\r\n") {
6582        return Err(HealthProbeError::bad_answer(
6583            "HTTP headers are incomplete or exceed 16 KiB",
6584        ));
6585    }
6586    line.truncate(line.len() - 2);
6587    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
6588}
6589
6590async fn probe_module_health(
6591    module_id: &str,
6592    runtime: &SupervisorRuntimeConfig,
6593    drain_deadline: Option<Instant>,
6594) -> Result<HealthReport, HealthProbeError> {
6595    probe_module_health_from(module_id, runtime, drain_deadline)
6596        .await
6597        .map(|(report, _)| report)
6598}
6599
6600/// [`probe_module_health`], also returning the connection whose registration
6601/// answered, so its answer is credited to that registration and no other.
6602async fn probe_module_health_from(
6603    module_id: &str,
6604    runtime: &SupervisorRuntimeConfig,
6605    drain_deadline: Option<Instant>,
6606) -> Result<(HealthReport, ConnectionId), HealthProbeError> {
6607    let Some(forwarding) = runtime.forwarding.as_ref() else {
6608        return Err(HealthProbeError::misconfigured(
6609            "supervisor was not configured with a forwarding table",
6610        ));
6611    };
6612    let probe_started_at = Instant::now();
6613    let mut deadline = probe_started_at + runtime.health.deadline;
6614    if let Some(drain_deadline) = drain_deadline {
6615        deadline = deadline.min(drain_deadline);
6616    }
6617    let pending = if drain_deadline.is_some() {
6618        forwarding.begin_drain_health_probe_rpc_for(
6619            module_id,
6620            MODULE_CONTROL_OP_HEALTH_CHECK,
6621            probe_started_at,
6622            deadline,
6623        )
6624    } else {
6625        forwarding.begin_health_probe_rpc_for(
6626            module_id,
6627            MODULE_CONTROL_OP_HEALTH_CHECK,
6628            probe_started_at,
6629            deadline,
6630        )
6631    }
6632    .map_err(|err| {
6633        // The endpoint is not registered, so there is no live control lane to
6634        // ask. That is the module being absent, not slow.
6635        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6636    })?;
6637    let probed = pending.endpoint.connection_id;
6638    await_health_probe(forwarding, pending, deadline, runtime.health.deadline)
6639        .await
6640        .map(|report| (report, probed))
6641}
6642
6643/// [`probe_module_health`] for one endpoint rather than the id's active one.
6644///
6645/// A swap probes two processes that no by-id lookup reaches: its candidate
6646/// before cutover, and its superseded incumbent (for busy gauges) while the
6647/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
6648/// bounds the by-id drain probe.
6649async fn probe_endpoint_health(
6650    endpoint: crate::ModuleEndpointId,
6651    runtime: &SupervisorRuntimeConfig,
6652    deadline_cap: Option<Instant>,
6653) -> Result<HealthReport, HealthProbeError> {
6654    let Some(forwarding) = runtime.forwarding.as_ref() else {
6655        return Err(HealthProbeError::misconfigured(
6656            "supervisor was not configured with a forwarding table",
6657        ));
6658    };
6659    let probe_started_at = Instant::now();
6660    let mut deadline = probe_started_at + runtime.health.deadline;
6661    if let Some(cap) = deadline_cap {
6662        deadline = deadline.min(cap);
6663    }
6664    let pending = forwarding
6665        .begin_endpoint_health_probe_rpc_for(
6666            endpoint,
6667            MODULE_CONTROL_OP_HEALTH_CHECK,
6668            probe_started_at,
6669            deadline,
6670        )
6671        .map_err(|err| {
6672            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6673        })?;
6674    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
6675}
6676
6677/// Send a begun health probe and classify its answer.
6678async fn await_health_probe(
6679    forwarding: &ForwardingTable,
6680    pending: PendingModuleControlRpc,
6681    deadline: Instant,
6682    probe_budget: Duration,
6683) -> Result<HealthReport, HealthProbeError> {
6684    let PendingModuleControlRpc {
6685        endpoint,
6686        module_sink,
6687        negotiated_ver,
6688        corr,
6689        receiver,
6690    } = pending;
6691    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6692        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
6693    })?;
6694    let frame = Frame::build_with_version(
6695        negotiated_ver,
6696        FrameType::Request,
6697        control_flags(),
6698        0,
6699        0,
6700        corr,
6701        body,
6702    )
6703    .map_err(|err| {
6704        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
6705    })?;
6706
6707    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
6708    // blocks waiting for capacity when the module's egress queue is full, and an
6709    // unbounded await here freezes the whole supervision actor (it stops polling
6710    // Child::wait and supervisor commands), making the module unrecoverable
6711    // in-band. On timeout the probe fails like any transport failure.
6712    match timeout_at(deadline, module_sink.send(frame)).await {
6713        Ok(Ok(())) => {}
6714        Ok(Err(err)) => {
6715            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6716            // A closed sink means the module's egress channel is gone -- the
6717            // receiving half is dropped when its connection tears down. Proof.
6718            return Err(HealthProbeError::lane_dead(format!(
6719                "failed to send health.check: {err}"
6720            )));
6721        }
6722        Err(_elapsed) => {
6723            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6724            // A full egress queue means the module is not draining its socket, which
6725            // is consistent with a wedged module AND with one whose reader is merely
6726            // starved. Silence, not proof.
6727            return Err(HealthProbeError::no_answer(
6728                "health.check send timed out before enqueue (module egress full)",
6729            ));
6730        }
6731    }
6732
6733    match timeout_at(deadline, receiver).await {
6734        Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => Err(HealthProbeError::bad_answer(
6735            "health.check returned an unexpected config response",
6736        )),
6737        // Each arm records WHAT WAS OBSERVED. Four of them are the module
6738        // demonstrably answering -- rejected, non-health, malformed, wrong op --
6739        // and those prove it is alive even though the probe failed.
6740        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6741            response.health_report().ok_or_else(|| {
6742                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
6743            })
6744        }
6745        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
6746            format!("health.check rejected: {}", body.message),
6747        )),
6748        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6749            Err(HealthProbeError::lane_dead(message))
6750        }
6751        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6752            Err(HealthProbeError::bad_answer(message))
6753        }
6754        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6755            Err(HealthProbeError::bad_answer(format!(
6756                "expected module-control op '{expected}', got '{actual}'"
6757            )))
6758        }
6759        // A reply that crosses the deadline before this waiter observes it is
6760        // still proof of life. The forwarding path records its end-to-end latency
6761        // before delivering this classification.
6762        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
6763            "module answered health.check after its daemon deadline",
6764        )),
6765        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
6766            "health.check waiter was canceled before the module responded",
6767        )),
6768        Err(_) => {
6769            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
6770            Err(HealthProbeError::no_answer(format!(
6771                "module did not answer health.check within {probe_budget:?}"
6772            )))
6773        }
6774    }
6775}
6776
6777/// Act on a health probe's answer. `probed` is the connection whose
6778/// registration answered (`None` for an HTTP probe, which reaches no
6779/// registration); only that registration's spawn is credited with readiness.
6780#[allow(clippy::too_many_arguments)]
6781async fn handle_health_report(
6782    spec: &ModuleSpec,
6783    runtime: &SupervisorRuntimeConfig,
6784    registry: &Registry,
6785    process_liveness: &SupervisorProcessLiveness,
6786    snapshot: &SharedSnapshot,
6787    child: &mut Option<SupervisedChild>,
6788    report: HealthReport,
6789    now_ms: u64,
6790    delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6791    probed: Option<ConnectionId>,
6792) {
6793    let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6794        (Some(delivery), Some(probe)) => {
6795            delivery
6796                .health_result(
6797                    &spec.module_id,
6798                    runtime.forwarding.as_deref(),
6799                    registry,
6800                    probe,
6801                    report.status == HealthStatus::Ok,
6802                )
6803                .await
6804        }
6805        _ => false,
6806    };
6807    if let (Some(delivery), Some(spawn)) = (
6808        &runtime.config_delivery,
6809        child
6810            .as_ref()
6811            .and_then(|child| child.config_delivery.as_ref()),
6812    ) {
6813        delivery
6814            .mark_readiness(
6815                &spec.module_id,
6816                runtime.forwarding.as_deref(),
6817                probed,
6818                spawn,
6819                report.status == HealthStatus::Ok,
6820            )
6821            .await;
6822    }
6823    let status = supervisor_health_status(report.status);
6824    let detail = report.detail.clone();
6825    let metrics = truncate_health_metrics(report.metrics);
6826    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6827        state.health.status = status;
6828        state.last_probe_evidence = None;
6829        state.health.last_probe_ms = Some(now_ms);
6830        state.health.detail = detail.clone();
6831        state.health.metrics = metrics.clone();
6832        state.health.consecutive_failures = 0;
6833        state.failed_retry.observe_health(
6834            report.status == HealthStatus::Ok,
6835            runtime.restart_policy.window,
6836        );
6837    });
6838
6839    if delivery_failed {
6840        return;
6841    }
6842    let action = match report.status {
6843        HealthStatus::Ok => return,
6844        HealthStatus::Degraded => runtime.health.on_degraded,
6845        HealthStatus::Failing => runtime.health.on_failing,
6846    };
6847    apply_l3_health_action(
6848        spec,
6849        runtime,
6850        registry,
6851        process_liveness,
6852        snapshot,
6853        child,
6854        status,
6855        detail.as_deref(),
6856        action,
6857        now_ms,
6858    )
6859    .await;
6860}
6861
6862#[allow(clippy::too_many_arguments)]
6863async fn handle_health_probe_failure(
6864    spec: &ModuleSpec,
6865    runtime: &SupervisorRuntimeConfig,
6866    registry: &Registry,
6867    process_liveness: &SupervisorProcessLiveness,
6868    snapshot: &SharedSnapshot,
6869    child: &mut Option<SupervisedChild>,
6870    err: HealthProbeError,
6871    now_ms: u64,
6872    delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6873) {
6874    let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6875        (Some(delivery), Some(probe)) => {
6876            delivery
6877                .health_result(
6878                    &spec.module_id,
6879                    runtime.forwarding.as_deref(),
6880                    registry,
6881                    probe,
6882                    false,
6883                )
6884                .await
6885        }
6886        _ => false,
6887    };
6888    let threshold = runtime.health.failure_threshold.max(1);
6889    let held = if err.evidence == HealthProbeEvidence::NoAnswer {
6890        runtime.host_health.record_late_module(&spec.module_id)
6891    } else {
6892        false
6893    };
6894    let mut failures = 0;
6895    // Carry the evidence class into the operator-visible detail. Without it,
6896    // "module did not answer within 5s" and "the control lane is gone" are two
6897    // prose strings in the same field, and the reader has to know the codebase to
6898    // tell which one is proof of anything.
6899    let detail = if held {
6900        format!("[{}] {err}; host overloaded", err.label())
6901    } else {
6902        format!("[{}] {err}", err.label())
6903    };
6904    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6905        state.last_probe_evidence = Some(err.evidence);
6906        state
6907            .failed_retry
6908            .observe_health(false, runtime.restart_policy.window);
6909        // A failed wire probe invalidates the last report, even before the
6910        // restart threshold. HTTP probes already mark failures as Failing.
6911        if held {
6912            state.health.status = SupervisorHealthStatus::Degraded;
6913        } else if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
6914            state.health.status = SupervisorHealthStatus::Unknown;
6915        }
6916        state.health.last_probe_ms = Some(now_ms);
6917        // Held silence breaks the streak: after the host recovers, only new
6918        // failures count. Replaying pre-overload misses would restart immediately.
6919        state.health.consecutive_failures = if held {
6920            0
6921        } else {
6922            state.health.consecutive_failures.saturating_add(1)
6923        };
6924        state.health.detail = Some(detail.clone());
6925        state.health.metrics = None;
6926        failures = state.health.consecutive_failures;
6927    });
6928
6929    if delivery_failed {
6930        return;
6931    }
6932    if held || failures < threshold {
6933        warn!(
6934            module_id = %spec.module_id,
6935            consecutive_failures = failures,
6936            threshold,
6937            evidence = err.label(),
6938            detail = %detail,
6939            "health.check probe failed"
6940        );
6941        return;
6942    }
6943
6944    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6945        state.state = ModuleState::Unresponsive;
6946        state.health.status = SupervisorHealthStatus::Unresponsive;
6947    });
6948    // The evidence class is logged at the kill site because this is the line an
6949    // operator reads after an unexplained restart. A streak of `no-answer` under
6950    // machine load is the known false-positive shape; a `lane-dead` is not.
6951    if runtime.health.critical {
6952        error!(
6953            module_id = %spec.module_id,
6954            status = "unresponsive",
6955            evidence = err.label(),
6956            detail = %detail,
6957            "critical module health alert"
6958        );
6959    } else {
6960        warn!(
6961            module_id = %spec.module_id,
6962            status = "unresponsive",
6963            evidence = err.label(),
6964            detail = %detail,
6965            "module health threshold breached"
6966        );
6967    }
6968    if let Err(err) = health_restart_child(
6969        spec,
6970        runtime,
6971        registry,
6972        process_liveness,
6973        snapshot,
6974        child,
6975        SupervisorHealthStatus::Unresponsive,
6976        Some(&detail),
6977        now_ms,
6978    )
6979    .await
6980    {
6981        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
6982    }
6983}
6984
6985#[allow(clippy::too_many_arguments)]
6986async fn apply_l3_health_action(
6987    spec: &ModuleSpec,
6988    runtime: &SupervisorRuntimeConfig,
6989    registry: &Registry,
6990    process_liveness: &SupervisorProcessLiveness,
6991    snapshot: &SharedSnapshot,
6992    child: &mut Option<SupervisedChild>,
6993    status: SupervisorHealthStatus,
6994    detail: Option<&str>,
6995    action: HealthAction,
6996    now_ms: u64,
6997) {
6998    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
6999    match action {
7000        HealthAction::Report => {
7001            info!(
7002                module_id = %spec.module_id,
7003                status = ?status,
7004                detail,
7005                "module reported non-ok health"
7006            );
7007        }
7008        HealthAction::Alert => {
7009            error!(
7010                module_id = %spec.module_id,
7011                status = ?status,
7012                detail,
7013                "module health alert"
7014            );
7015        }
7016        HealthAction::Restart => {
7017            if let Err(err) = health_restart_child(
7018                spec,
7019                runtime,
7020                registry,
7021                process_liveness,
7022                snapshot,
7023                child,
7024                status,
7025                detail,
7026                now_ms,
7027            )
7028            .await
7029            {
7030                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
7031            }
7032        }
7033    }
7034}
7035
7036#[allow(clippy::too_many_arguments)]
7037async fn health_restart_child(
7038    spec: &ModuleSpec,
7039    runtime: &SupervisorRuntimeConfig,
7040    registry: &Registry,
7041    process_liveness: &SupervisorProcessLiveness,
7042    snapshot: &SharedSnapshot,
7043    child: &mut Option<SupervisedChild>,
7044    status: SupervisorHealthStatus,
7045    detail: Option<&str>,
7046    now_ms: u64,
7047) -> Result<(), SuperviseError> {
7048    budgeted_restart_child(
7049        spec,
7050        runtime,
7051        registry,
7052        process_liveness,
7053        snapshot,
7054        child,
7055        status,
7056        detail,
7057        now_ms,
7058        true,
7059    )
7060    .await
7061}
7062
7063#[allow(clippy::too_many_arguments)]
7064async fn budgeted_restart_child(
7065    spec: &ModuleSpec,
7066    runtime: &SupervisorRuntimeConfig,
7067    registry: &Registry,
7068    process_liveness: &SupervisorProcessLiveness,
7069    snapshot: &SharedSnapshot,
7070    child: &mut Option<SupervisedChild>,
7071    status: SupervisorHealthStatus,
7072    detail: Option<&str>,
7073    now_ms: u64,
7074    health_failure: bool,
7075) -> Result<(), SuperviseError> {
7076    if let Some(error) = runtime
7077        .config_delivery
7078        .as_ref()
7079        .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
7080        .filter(|error| error.code != "config_delivery_deferred")
7081    {
7082        return Err(config_delivery::gate_hold_error(
7083            spec,
7084            error,
7085            runtime.config_delivery.as_ref(),
7086        ));
7087    }
7088    let recovery_reason = if health_failure {
7089        "health_check"
7090    } else {
7091        "configuration_delivery"
7092    };
7093    let (enabled, schedule) = {
7094        let mut state = lock_snapshot(snapshot)?;
7095        let enabled = state.enabled;
7096        let schedule = if enabled {
7097            state.next_crash_restart(&runtime.restart_policy, Instant::now())
7098        } else {
7099            None
7100        };
7101        (enabled, schedule)
7102    };
7103
7104    if !enabled {
7105        return Err(SuperviseError::Disabled {
7106            module_id: spec.module_id.clone(),
7107        });
7108    }
7109
7110    if schedule.is_none() {
7111        if health_failure {
7112            record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
7113        }
7114        error!(
7115            module_id = %spec.module_id,
7116            status = ?status,
7117            detail,
7118            max_restarts = runtime.restart_policy.max_restarts,
7119            window_secs = runtime.restart_policy.window.as_secs(),
7120            reason = %runtime.restart_policy.budget_exhausted_detail(),
7121            recovery_reason,
7122            "automatic restart budget exhausted; marking module failed"
7123        );
7124        let stop_notice = begin_forwarding_drain_if_configured(
7125            spec,
7126            runtime,
7127            registry,
7128            snapshot,
7129            Some(true),
7130            RouteCloseReason::Disable,
7131        )
7132        .await?;
7133        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7134            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
7135        })?;
7136        let result = drain_optional_child(
7137            &spec.module_id,
7138            spec.protocol,
7139            stop_notice,
7140            registry,
7141            runtime.forwarding.as_deref(),
7142            snapshot,
7143            &runtime.terminal_ring,
7144            &runtime.spawn_events,
7145            child,
7146            runtime.drain_timeout,
7147            ModuleState::Failed,
7148            Some(true),
7149            &runtime.failure_context(true),
7150        )
7151        .await;
7152        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7153            if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
7154                state.failed_retry.schedule(&spec.module_id);
7155            }
7156        })?;
7157        result?;
7158        process_liveness.untrack_if_current(&spec.module_id, snapshot);
7159        return Ok(());
7160    }
7161
7162    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
7163    let mut restart_count = 0;
7164    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7165        restart_count = state.crash_restarts.len();
7166        state.state = ModuleState::Unresponsive;
7167        if health_failure {
7168            state.health.status = status;
7169            state.health.last_action = Some(HealthAction::Restart.to_string());
7170            state.health.last_action_ms = Some(now_ms);
7171        }
7172    })?;
7173    warn!(
7174        module_id = %spec.module_id,
7175        status = ?status,
7176        detail,
7177        restart_count,
7178        restart_in_window = schedule.restart_in_window,
7179        delay_ms = schedule.delay.as_millis() as u64,
7180        recovery_reason,
7181        "automatic module restart"
7182    );
7183
7184    let stop_notice = begin_forwarding_drain_if_configured(
7185        spec,
7186        runtime,
7187        registry,
7188        snapshot,
7189        Some(true),
7190        RouteCloseReason::Restart,
7191    )
7192    .await?;
7193    drain_optional_child(
7194        &spec.module_id,
7195        spec.protocol,
7196        stop_notice,
7197        registry,
7198        runtime.forwarding.as_deref(),
7199        snapshot,
7200        &runtime.terminal_ring,
7201        &runtime.spawn_events,
7202        child,
7203        runtime.drain_timeout,
7204        ModuleState::Restarting,
7205        Some(true),
7206        &runtime.failure_context(true),
7207    )
7208    .await?;
7209    schedule_respawn(
7210        runtime,
7211        snapshot,
7212        &spec.module_id,
7213        schedule.delay,
7214        RespawnKind::Spawn,
7215        true,
7216    )
7217}
7218
7219fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
7220    if let Some(reply) = runtime
7221        .deferred_reload_reply
7222        .lock()
7223        .unwrap_or_else(|p| p.into_inner())
7224        .take()
7225    {
7226        let _ = reply.send(Err(SuperviseError::ReloadFailed {
7227            module_id: module_id.to_string(),
7228            reason: reason.to_string(),
7229        }));
7230    }
7231}
7232
7233fn park_config_spawn(
7234    spec: &ModuleSpec,
7235    runtime: &SupervisorRuntimeConfig,
7236    snapshot: &SharedSnapshot,
7237    failure: &FailureContext,
7238    admission_refused: Option<String>,
7239) -> Result<(), SuperviseError> {
7240    *runtime
7241        .scheduled_respawn
7242        .lock()
7243        .unwrap_or_else(|poisoned| poisoned.into_inner()) = None;
7244    if let Some(reply) = runtime
7245        .deferred_reload_reply
7246        .lock()
7247        .unwrap_or_else(|poisoned| poisoned.into_inner())
7248        .take()
7249    {
7250        let _ = reply.send(Err(SuperviseError::ConfigHeld {
7251            module_id: spec.module_id.clone(),
7252            code: "config_gated".into(),
7253            detail: "module is held by configuration admission".into(),
7254        }));
7255    }
7256    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7257        state.admission_refused = admission_refused;
7258        state.config_hold_daemon_initiated = failure.daemon_initiated;
7259        state.config_hold_approval = failure.approval.clone();
7260        state.config_hold_parked = true;
7261        state.state = ModuleState::Stopped;
7262        state.respawn_pending = false;
7263        state.failed_retry.pending = None;
7264        // Parking ends this attempt and discards its failure timestamp. If a
7265        // later release fails, its retry delay starts at that new failure.
7266        state.failed_retry.failed_at = None;
7267        clear_current_process_facts(state);
7268    })
7269}
7270
7271fn try_park_config_attempt(
7272    spec: &ModuleSpec,
7273    runtime: &SupervisorRuntimeConfig,
7274    snapshot: &SharedSnapshot,
7275    failure: &FailureContext,
7276    program_base: Option<&subc_config::Digest>,
7277) -> Result<bool, SuperviseError> {
7278    let result = (|| {
7279        if !lock_snapshot(snapshot)?.enabled {
7280            return Ok(false);
7281        }
7282        #[cfg(test)]
7283        if let Some(delivery) = &runtime.config_delivery {
7284            delivery.0.pause_before_config_park_for_test();
7285        }
7286        let Some(delivery) = &runtime.config_delivery else {
7287            return Ok(false);
7288        };
7289        let parked = delivery
7290            .0
7291            .with_recorded_admission_hold(&spec.module_id, program_base, |refusal| {
7292                park_config_spawn(spec, runtime, snapshot, failure, refusal)
7293            })
7294            .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(delivery)))?;
7295        if let Some(result) = parked {
7296            result?;
7297            Ok(true)
7298        } else {
7299            Ok(false)
7300        }
7301    })();
7302    if let Err(error) = &result {
7303        // A failed park still belongs to the attempt that tried to park. Mark it
7304        // Failed here, before the monitor can use a daemon/operator origin saved
7305        // for a different attempt to decide whether to schedule a slow retry.
7306        fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7307        record_approved_attempt_failure(spec, snapshot, failure, error);
7308    }
7309    result
7310}
7311
7312fn record_approved_attempt_failure(
7313    spec: &ModuleSpec,
7314    snapshot: &SharedSnapshot,
7315    failure: &FailureContext,
7316    error: &SuperviseError,
7317) {
7318    let Some(approval) = &failure.approval else {
7319        return;
7320    };
7321    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7322        if state.enabled
7323            && state.state == ModuleState::Failed
7324            && state
7325                .approved_start_failure
7326                .as_ref()
7327                .is_none_or(|(identity, _)| identity != approval)
7328        {
7329            state.approved_start_failure = Some((approval.clone(), error.to_string()));
7330        }
7331    });
7332}
7333
7334fn hold_config_spawn(
7335    spec: &ModuleSpec,
7336    runtime: &SupervisorRuntimeConfig,
7337    snapshot: &SharedSnapshot,
7338    failure: &FailureContext,
7339) -> Result<(), SuperviseError> {
7340    if !lock_snapshot(snapshot)?.enabled {
7341        return Ok(());
7342    }
7343    if !try_park_config_attempt(spec, runtime, snapshot, failure, None)? {
7344        // This caller consumed an actual start attempt. If its hold ended before
7345        // parking, retain that attempt instead of waiting for a lost release.
7346        defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
7347    }
7348    Ok(())
7349}
7350
7351fn schedule_respawn(
7352    runtime: &SupervisorRuntimeConfig,
7353    snapshot: &SharedSnapshot,
7354    module_id: &str,
7355    delay: Duration,
7356    kind: RespawnKind,
7357    daemon_initiated: bool,
7358) -> Result<(), SuperviseError> {
7359    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
7360    update_snapshot(snapshot, Some(module_id), |state| {
7361        state.respawn_pending = true
7362    })?;
7363    *runtime
7364        .scheduled_respawn
7365        .lock()
7366        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7367        deadline: Instant::now() + delay,
7368        kind,
7369        daemon_initiated,
7370        approval: None,
7371    });
7372    Ok(())
7373}
7374
7375const CONFIG_ADMISSION_RETRY_DELAY: Duration = Duration::from_millis(100);
7376
7377fn defer_config_spawn(
7378    spec: &ModuleSpec,
7379    runtime: &SupervisorRuntimeConfig,
7380    snapshot: &SharedSnapshot,
7381    kind: RespawnKind,
7382    failure: &FailureContext,
7383) -> Result<(), SuperviseError> {
7384    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7385        state.state = ModuleState::Restarting;
7386        state.respawn_pending = true;
7387        clear_current_process_facts(state);
7388    })?;
7389    *runtime
7390        .scheduled_respawn
7391        .lock()
7392        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7393        deadline: Instant::now() + CONFIG_ADMISSION_RETRY_DELAY,
7394        kind,
7395        daemon_initiated: failure.daemon_initiated,
7396        approval: failure.approval.clone(),
7397    });
7398    Ok(())
7399}
7400
7401fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
7402    let _ = update_snapshot(snapshot, Some(module_id), |state| {
7403        state.health.last_action = Some(action);
7404        state.health.last_action_ms = Some(now_ms);
7405    });
7406}
7407
7408fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
7409    match status {
7410        HealthStatus::Ok => SupervisorHealthStatus::Ok,
7411        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
7412        HealthStatus::Failing => SupervisorHealthStatus::Failing,
7413    }
7414}
7415
7416/// Caps the metrics blob stored in the cached supervisor snapshot, which is
7417/// returned to every `supervisor.list` and `supervisor.health` caller.
7418///
7419/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
7420/// path: that request exists to return a module's complete metrics object, and
7421/// `ck health <module-id>` documents it as the way to see what the cached view
7422/// truncates. The asymmetry is the feature.
7423///
7424/// So a new caller must decide which side it is on rather than assume the cap is
7425/// universal. Reaching for it on a fresh-probe path would silently reintroduce
7426/// the truncation that path exists to avoid.
7427fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
7428    let metrics = metrics?;
7429    match serde_json::to_vec(&metrics) {
7430        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
7431            "truncated": true,
7432            "original_bytes": encoded.len(),
7433        })),
7434        Ok(_) | Err(_) => Some(metrics),
7435    }
7436}
7437
7438/// Spread health probes so a fleet-wide restart does not converge them.
7439///
7440/// The delay is derived from the module id and probe index rather than a random
7441/// source, so it is deterministic per module: a module keeps its own offset
7442/// across daemon restarts instead of re-rolling into a collision.
7443fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
7444    if cadence.is_zero() {
7445        return Duration::ZERO;
7446    }
7447    let cadence_ms = cadence.as_millis() as u64;
7448    // This early return is REDUNDANT, deliberately, and a mutation run will show
7449    // it surviving removal. Recording why here so the next person to notice does
7450    // not have to re-derive it:
7451    //
7452    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
7453    //   a zero cadence and builds the Duration from whole milliseconds, so a
7454    //   sub-millisecond cadence cannot come from config.
7455    // - Even if reached it changes no answer. The `.max(1)` below makes the span
7456    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
7457    //   -- exactly what this returns.
7458    //
7459    // Kept as a guard against a future widening of the config parser (accepting
7460    // microseconds, say), which would make the sub-millisecond case reachable.
7461    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
7462    // divides by zero. Remove this and nothing changes.
7463    if cadence_ms == 0 {
7464        return cadence;
7465    }
7466    // Note that this never returns less than one cadence, including for the FIRST
7467    // probe. So a freshly registered module reports health `unknown` for a full
7468    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
7469    // ready to answer.
7470    //
7471    // That is a property of the supervisor's schedule, not of any module: an
7472    // operator watching a restart sees `unknown` and cannot tell it from a module
7473    // that is slow to warm. Measured on two unrelated modules, both flipping to
7474    // `ok` between 22s and 32s after restart.
7475    //
7476    // Left as-is because spreading the first probe is what keeps a fleet-wide
7477    // restart from firing fourteen simultaneous probes into a cold machine. The
7478    // alternative -- probe at t+0 and jitter only from the second onward -- trades
7479    // that thundering herd for a faster first reading.
7480    let jitter_span = (cadence_ms / 10).max(1);
7481    let hash = module_id.as_bytes().iter().fold(
7482        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
7483        |acc, byte| {
7484            acc.wrapping_mul(1099511628211)
7485                .wrapping_add(u64::from(*byte))
7486        },
7487    );
7488    cadence + Duration::from_millis(hash % jitter_span)
7489}
7490
7491#[cfg(test)]
7492mod tests {
7493    use super::*;
7494
7495    #[test]
7496    fn registration_retirement_self_prefixed_owner_is_not_a_prefix_submodule() {
7497        let handle = SupervisorHandle::new();
7498        handle.set_spawn_nonce("prefix:owner", "owner-nonce".into());
7499        handle.set_reserved_prefixes("prefix:owner", &["prefix:".into()]);
7500        let owner = handle
7501            .reserved_hello_verification("prefix:owner", Some("owner-nonce"))
7502            .unwrap();
7503        assert!(owner.nonce_verified);
7504        assert!(
7505            owner.prefix_owner.is_none(),
7506            "an unreserved programmatic owner is not its own prefix child"
7507        );
7508        let child = handle
7509            .reserved_hello_verification("prefix:child", Some("owner-nonce"))
7510            .unwrap();
7511        assert!(child.nonce_verified);
7512        assert_eq!(child.prefix_owner.as_deref(), Some("prefix:owner"));
7513    }
7514
7515    #[test]
7516    fn readding_a_module_clears_its_rescan_removal_tombstone() {
7517        let handle = SupervisorHandle::new();
7518        let module_id = "readded-tombstone";
7519        handle.record_rescan_removal(module_id);
7520        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
7521
7522        handle.apply_identity_configuration(&ModuleSpec {
7523            script: None,
7524            module_id: module_id.to_string(),
7525            program: PathBuf::from("/test/module"),
7526            args: Vec::new(),
7527            env: Vec::new(),
7528            reserved: false,
7529            reserved_prefixes: Vec::new(),
7530            protocol: ModuleProtocol::Subc,
7531            overlap: Default::default(),
7532            carry_containment: false,
7533        });
7534
7535        assert!(
7536            handle.removal_tombstone_age_ms(module_id).is_none(),
7537            "a re-added module must not retain a stale removal tombstone"
7538        );
7539    }
7540
7541    /// What one module's owner looked like from the control plane at the
7542    /// instant after its first process was spawned.
7543    #[derive(Debug, PartialEq, Eq)]
7544    struct OwnerInSpawnWindow {
7545        module_id: String,
7546        configured: bool,
7547        on_roster: bool,
7548        admission_refusal: Option<&'static str>,
7549    }
7550
7551    /// A supervised module's process can connect, register, sync its scopes
7552    /// and describe them as soon as it is spawned. A plain `spawn` does not put
7553    /// the module on the roster until afterward; configured supervision does
7554    /// so before spawning. In both cases the owner must
7555    /// already read as configured, so a scoped `route.open` against it is
7556    /// refused as retryable `scope_not_synced` and not as terminal
7557    /// `scope_not_live` ("will never sync").
7558    ///
7559    /// The hook runs just after each initial spawn attempt, before returning to
7560    /// the caller, so no race with a real child's registration is needed.
7561    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7562    async fn a_new_module_is_configured_before_its_first_process_can_register() {
7563        use crate::scopes::ScopeTable;
7564        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
7565
7566        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
7567        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
7568            script: None,
7569            module_id: module_id.to_string(),
7570            program,
7571            args: Vec::new(),
7572            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7573                .into_iter()
7574                .map(|key| (key.to_string(), dir.path().display().to_string()))
7575                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7576                .collect(),
7577            reserved: false,
7578            reserved_prefixes: Vec::new(),
7579            protocol: ModuleProtocol::Subc,
7580            overlap: Default::default(),
7581            carry_containment: false,
7582        };
7583        let live = super::terminal_history_tests::fake_aft_stub_path();
7584        let missing = dir.path().join("definitely-missing-module");
7585
7586        let handle = SupervisorHandle::new();
7587        let mut supervisor =
7588            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7589                .with_handle(handle.clone());
7590        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
7591        let hook_handle = handle.clone();
7592        let hook_observed = Arc::clone(&observed);
7593        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7594            // Exactly what the control plane computes for a scoped route.open
7595            // naming this module as the owner of a scope it has not synced.
7596            let configured = hook_handle.is_configured(module_id);
7597            let selector = ScopeSelector::new(
7598                Principal::Reserved {
7599                    module_id: module_id.to_string(),
7600                },
7601                "s".to_string(),
7602                Some(1),
7603            );
7604            let carrier = Principal::Reserved {
7605                module_id: "carrier".to_string(),
7606            };
7607            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
7608                .admit(&carrier, module_id, &selector, configured)
7609            {
7610                Ok(_) => None,
7611                Err(refusal) => Some(refusal.code),
7612            };
7613            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
7614                module_id: module_id.to_string(),
7615                configured,
7616                on_roster: hook_handle.get(module_id).is_some(),
7617                admission_refusal,
7618            });
7619        })));
7620
7621        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
7622        let configured = supervisor
7623            .supervise_configured(stub("configured", live.clone()), true)
7624            .unwrap();
7625        let with_health = supervisor
7626            .supervise_configured_with_health(
7627                stub("with-health", live.clone()),
7628                true,
7629                HealthConfig::default(),
7630                None,
7631                RestartPolicy::default(),
7632            )
7633            .unwrap();
7634        // The failed-spawn path still puts the module on the roster (as
7635        // failed), so it is configured throughout.
7636        let failed = supervisor
7637            .supervise_configured_with_health(
7638                stub("failed-spawn", missing.clone()),
7639                true,
7640                HealthConfig::default(),
7641                None,
7642                RestartPolicy::default(),
7643            )
7644            .unwrap();
7645        // A failed plain `spawn` puts nothing on the roster, so its mark is
7646        // taken back once the spawn has failed.
7647        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
7648
7649        let expected = [
7650            "plain",
7651            "configured",
7652            "with-health",
7653            "failed-spawn",
7654            "spawn-error",
7655        ]
7656        .into_iter()
7657        .map(|module_id| OwnerInSpawnWindow {
7658            module_id: module_id.to_string(),
7659            configured: true,
7660            on_roster: !matches!(module_id, "plain" | "spawn-error"),
7661            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
7662        })
7663        .collect::<Vec<_>>();
7664        assert_eq!(*observed.lock().unwrap(), expected);
7665
7666        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
7667            assert!(
7668                handle.get(module_id).is_some(),
7669                "{module_id} is on the roster"
7670            );
7671            assert!(
7672                handle.is_configured(module_id),
7673                "{module_id} stays configured"
7674            );
7675        }
7676        assert!(handle.get("spawn-error").is_none());
7677        assert!(
7678            !handle.is_configured("spawn-error"),
7679            "a plain spawn that failed must not leave its module marked configured"
7680        );
7681
7682        // Leaving the roster clears the mark with it.
7683        handle.retire("failed-spawn");
7684        assert!(!handle.is_configured("failed-spawn"));
7685
7686        for module in [plain, configured, with_health] {
7687            module.stop().await.unwrap();
7688        }
7689        drop(failed);
7690    }
7691
7692    /// A configured module reads as starting until the supervisor has admitted
7693    /// its first process (exec confirmed, identity durably recorded), even
7694    /// though that process is already running by then and can register over
7695    /// HELLO, which is what lists it in catalog.list. So a client that needs
7696    /// the supervisor state must wait for that state; seeing the module in
7697    /// the catalog does not mean supervisor.list already says running.
7698    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7699    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
7700        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
7701        let spec = ModuleSpec {
7702            script: None,
7703            module_id: "spawned-not-admitted".to_string(),
7704            program: super::terminal_history_tests::fake_aft_stub_path(),
7705            args: Vec::new(),
7706            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7707                .into_iter()
7708                .map(|key| (key.to_string(), dir.path().display().to_string()))
7709                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7710                .collect(),
7711            reserved: false,
7712            reserved_prefixes: Vec::new(),
7713            protocol: ModuleProtocol::Subc,
7714            overlap: Default::default(),
7715            carry_containment: false,
7716        };
7717        let handle = SupervisorHandle::new();
7718        let mut supervisor =
7719            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7720                .with_handle(handle.clone());
7721        let observed = Arc::new(Mutex::new(None));
7722        let hook_handle = handle.clone();
7723        let hook_observed = Arc::clone(&observed);
7724        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7725            let state = hook_handle
7726                .get(module_id)
7727                .expect("a configured module is on the roster before it is spawned")
7728                .state()
7729                .unwrap();
7730            *hook_observed.lock().unwrap() = Some(state);
7731        })));
7732
7733        let module = supervisor.supervise_configured(spec, true).unwrap();
7734        assert_eq!(
7735            *observed.lock().unwrap(),
7736            Some(ModuleState::Starting),
7737            "between the spawn and its admission the supervisor still reports starting"
7738        );
7739        assert_eq!(module.state().unwrap(), ModuleState::Running);
7740        module.stop().await.unwrap();
7741    }
7742
7743    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
7744        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
7745        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
7746            snapshot.process_alive = true;
7747            snapshot.pid = Some(41);
7748            snapshot.spawned_at_ms = Some(42);
7749            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
7750            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
7751            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
7752                device: 43,
7753                inode: 44,
7754                #[cfg(windows)]
7755                windows_image: None,
7756            });
7757        })
7758        .unwrap();
7759        snapshot
7760    }
7761
7762    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
7763        let snapshot = lock_snapshot(snapshot).unwrap();
7764        assert!(!snapshot.process_alive);
7765        assert_eq!(snapshot.pid, None);
7766        assert_eq!(snapshot.spawned_at_ms, None);
7767        assert_eq!(snapshot.spawned_from, None);
7768        assert_eq!(snapshot.launch_spec_digest, None);
7769        assert_eq!(snapshot.spawned_file_identity, None);
7770    }
7771
7772    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7773    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
7774        let supervisor =
7775            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7776        let mut runtime = supervisor.runtime_config();
7777        runtime.test_seed_stale_facts_before_enable_spawn = true;
7778        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
7779        let mut child = None;
7780        let spec = ModuleSpec {
7781            script: None,
7782            module_id: "failed-enable-clears-facts".to_string(),
7783            program: PathBuf::from("/definitely/missing/failed-enable-module"),
7784            args: Vec::new(),
7785            env: Vec::new(),
7786            reserved: false,
7787            reserved_prefixes: Vec::new(),
7788            protocol: ModuleProtocol::Subc,
7789            overlap: Default::default(),
7790            carry_containment: false,
7791        };
7792
7793        let result = set_child_enabled(
7794            &spec,
7795            &runtime,
7796            &supervisor.registry,
7797            &supervisor.process_liveness,
7798            &snapshot,
7799            &mut child,
7800            true,
7801            &runtime.failure_context(false),
7802        )
7803        .await;
7804
7805        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
7806        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7807        assert_snapshot_process_facts_cleared(&snapshot);
7808    }
7809
7810    #[tokio::test]
7811    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
7812        let supervisor =
7813            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7814        let runtime = supervisor.runtime_config();
7815        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
7816            ModuleState::Restarting,
7817            true,
7818        )));
7819        let spec = ModuleSpec {
7820            script: None,
7821            module_id: "start-stranded-restarting".to_string(),
7822            program: super::terminal_history_tests::fake_aft_stub_path(),
7823            args: Vec::new(),
7824            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
7825            reserved: false,
7826            reserved_prefixes: Vec::new(),
7827            protocol: ModuleProtocol::None,
7828            overlap: Default::default(),
7829            carry_containment: false,
7830        };
7831        let mut child = None;
7832        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
7833        assert!(!super::set_child_enabled(
7834            &spec,
7835            &runtime,
7836            &Registry::default(),
7837            &supervisor.process_liveness,
7838            &snapshot,
7839            &mut child,
7840            true,
7841            &runtime.failure_context(false),
7842        )
7843        .await
7844        .unwrap());
7845        assert!(child.is_none());
7846        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
7847        assert!(super::set_child_enabled(
7848            &spec,
7849            &runtime,
7850            &Registry::default(),
7851            &supervisor.process_liveness,
7852            &snapshot,
7853            &mut child,
7854            true,
7855            &runtime.failure_context(false),
7856        )
7857        .await
7858        .unwrap());
7859        assert_eq!(
7860            lock_snapshot(&snapshot).unwrap().state,
7861            ModuleState::Running
7862        );
7863        let mut child = child.unwrap();
7864        child.start_kill().unwrap();
7865        child.wait().await.unwrap();
7866    }
7867
7868    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7869    async fn failed_reload_spawn_clears_current_process_facts() {
7870        let supervisor =
7871            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7872        let mut runtime = supervisor.runtime_config();
7873        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
7874        let snapshot = stale_process_snapshot(ModuleState::Running, true);
7875        let mut child = None;
7876        let spec = ModuleSpec {
7877            script: None,
7878            module_id: "failed-reload-clears-facts".to_string(),
7879            program: PathBuf::from("/unused/failed-reload-module"),
7880            args: Vec::new(),
7881            env: Vec::new(),
7882            reserved: false,
7883            reserved_prefixes: Vec::new(),
7884            protocol: ModuleProtocol::Subc,
7885            overlap: Default::default(),
7886            carry_containment: false,
7887        };
7888
7889        let result = handle_reload_spawn_failure(
7890            &spec,
7891            &runtime,
7892            &supervisor.process_liveness,
7893            &snapshot,
7894            &mut child,
7895            "forced reload spawn failure".to_string(),
7896        )
7897        .await;
7898
7899        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
7900        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7901        assert_snapshot_process_facts_cleared(&snapshot);
7902    }
7903
7904    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7905    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
7906        let supervisor =
7907            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7908        let snapshot = stale_process_snapshot(ModuleState::Running, true);
7909        let module = supervisor.supervised_module(
7910            ModuleSpec {
7911                script: None,
7912                module_id: "drop-clears-facts".to_string(),
7913                program: PathBuf::from("/unused/drop-module"),
7914                args: Vec::new(),
7915                env: Vec::new(),
7916                reserved: false,
7917                reserved_prefixes: Vec::new(),
7918                protocol: ModuleProtocol::Subc,
7919                overlap: Default::default(),
7920                carry_containment: false,
7921            },
7922            supervisor.runtime_config(),
7923            Arc::clone(&snapshot),
7924            None,
7925        );
7926        assert!(!module
7927            .inner
7928            .monitor
7929            .lock()
7930            .unwrap()
7931            .as_ref()
7932            .unwrap()
7933            .is_finished());
7934
7935        drop(module);
7936
7937        assert_eq!(
7938            lock_snapshot(&snapshot).unwrap().state,
7939            ModuleState::Stopped
7940        );
7941        assert_snapshot_process_facts_cleared(&snapshot);
7942    }
7943
7944    #[cfg(unix)]
7945    #[tokio::test]
7946    async fn rescan_preserves_running_protocol_until_respawn() {
7947        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
7948        let initial = ModuleSpec {
7949            script: None,
7950            module_id: "rescan-protocol".into(),
7951            program: PathBuf::from("/bin/sleep"),
7952            args: vec!["60".into()],
7953            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7954                .into_iter()
7955                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
7956                .collect(),
7957            reserved: false,
7958            reserved_prefixes: vec![],
7959            protocol: ModuleProtocol::None,
7960            overlap: Default::default(),
7961            carry_containment: false,
7962        };
7963        let supervisor =
7964            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7965        let module = supervisor.spawn(initial.clone()).unwrap();
7966        assert!(module.status().unwrap().live);
7967        let mut next = initial;
7968        next.protocol = ModuleProtocol::Subc;
7969        module
7970            .update_configuration(next.clone(), HealthConfig::default(), None)
7971            .await
7972            .unwrap();
7973        assert!(
7974            module.status().unwrap().live,
7975            "rescan must not require HELLO from the old non-wire process"
7976        );
7977        let runtime = supervisor.runtime_config();
7978        let action = on_child_exit(
7979            &next,
7980            RestartPolicy::default(),
7981            &supervisor.registry,
7982            &module.inner.snapshot,
7983            &runtime.terminal_ring,
7984            &runtime.spawn_events,
7985            &runtime.child_roster,
7986            ExitReport {
7987                kind: ExitKind::Clean,
7988                code: Some(0),
7989                signal: None,
7990                at_ms: unix_ms_now(),
7991            },
7992            true,
7993        )
7994        .await;
7995        assert!(
7996            matches!(action, NextAction::Restart { .. }),
7997            "the old non-wire process's clean exit must restart"
7998        );
7999        module.drain().await.unwrap();
8000    }
8001
8002    #[cfg(unix)]
8003    #[tokio::test]
8004    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
8005        use std::os::unix::fs::PermissionsExt;
8006        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
8007        let script = dir.join("module.sh");
8008        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
8009        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
8010        let record_path = dir.join("live-children.json");
8011        let supervisor =
8012            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8013                .with_live_children_record(&record_path);
8014        for (program, args) in [
8015            (PathBuf::from("sleep"), vec!["60".into()]),
8016            (script, vec![]),
8017        ] {
8018            let spec = ModuleSpec {
8019                script: None,
8020                module_id: "image-identity".into(),
8021                program: program.clone(),
8022                args,
8023                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
8024                    .into_iter()
8025                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
8026                    .collect(),
8027                reserved: false,
8028                reserved_prefixes: vec![],
8029                protocol: ModuleProtocol::None,
8030                overlap: Default::default(),
8031                carry_containment: false,
8032            };
8033            let module = supervisor.spawn(spec).unwrap();
8034            #[cfg(target_os = "macos")]
8035            {
8036                // SETEXEC confirmation is asynchronous; the orphan record must
8037                // identify the final image, never the intermediate trampoline.
8038                let deadline = Instant::now() + Duration::from_secs(5);
8039                while crate::live_children::read_record(&record_path)
8040                    .unwrap()
8041                    .iter()
8042                    .all(|entry| entry.executable.is_none())
8043                {
8044                    assert!(Instant::now() < deadline, "module image was not confirmed");
8045                    tokio::time::sleep(Duration::from_millis(5)).await;
8046                }
8047            }
8048            let entry = crate::live_children::read_record(&record_path)
8049                .unwrap()
8050                .pop()
8051                .unwrap();
8052            let observed = subc_os::Process::open(entry.pid)
8053                .unwrap()
8054                .unwrap()
8055                .observe()
8056                .unwrap();
8057            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
8058            module.drain().await.unwrap();
8059            assert_eq!(
8060                verdict,
8061                crate::live_children::IdentityVerdict::Matches,
8062                "program {program:?}: recorded {entry:?}, observed {observed:?}"
8063            );
8064        }
8065    }
8066
8067    #[cfg(unix)]
8068    fn http_fixture(
8069        dir: &std::path::Path,
8070        url: &str,
8071        threshold: u32,
8072    ) -> crate::daemon_config::ConfiguredModule {
8073        let path = dir.join("subc.jsonc");
8074        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
8075            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
8076            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
8077            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
8078            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
8079        }}}).to_string()).unwrap();
8080        crate::daemon_config::load(&path)
8081            .unwrap()
8082            .unwrap()
8083            .modules
8084            .pop()
8085            .unwrap()
8086    }
8087
8088    #[cfg(unix)]
8089    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
8090        timeout(Duration::from_secs(5), async {
8091            loop {
8092                if module.status().unwrap().health.status == status {
8093                    break;
8094                }
8095                sleep(Duration::from_millis(5)).await;
8096            }
8097        })
8098        .await
8099        .unwrap_or_else(|_| {
8100            panic!(
8101                "expected {status:?}, got {:?}",
8102                module.status().unwrap().health
8103            )
8104        });
8105    }
8106
8107    #[cfg(unix)]
8108    #[tokio::test]
8109    async fn http_health_status_flips_ok_failing_ok() {
8110        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8111        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
8112        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8113        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
8114        let serving_status = status.clone();
8115        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8116        let server = tokio::spawn(async move {
8117            loop {
8118                let (mut stream, _) = listener.accept().await.unwrap();
8119                let mut request = [0u8; 2048];
8120                let count = stream.read(&mut request).await.unwrap();
8121                assert!(count > 0, "a probe must send an HTTP request");
8122                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
8123                let body = if code == 200 {
8124                    "ready"
8125                } else {
8126                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
8127                };
8128                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
8129                let _ = stream.write_all(response.as_bytes()).await;
8130            }
8131        });
8132        let configured = http_fixture(&dir, &url, 1000);
8133        let module =
8134            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8135                .supervise_configured_with_health(
8136                    configured.module_spec(),
8137                    true,
8138                    configured.health,
8139                    configured.drain_timeout_ms,
8140                    configured.restart,
8141                )
8142                .unwrap();
8143        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8144        status.store(503, std::sync::atomic::Ordering::SeqCst);
8145        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
8146        assert!(module
8147            .status()
8148            .unwrap()
8149            .health
8150            .detail
8151            .unwrap()
8152            .contains("scratch failure"));
8153        status.store(200, std::sync::atomic::Ordering::SeqCst);
8154        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8155        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
8156        let before = module.status().unwrap();
8157        let (spec, mut health) = module.configuration().unwrap();
8158        health.http = None;
8159        module
8160            .update_configuration(spec.clone(), health.clone(), Some(10))
8161            .await
8162            .unwrap();
8163        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
8164        health.http = Some(url);
8165        module
8166            .update_configuration(spec, health, Some(10))
8167            .await
8168            .unwrap();
8169        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8170        assert_eq!(
8171            module.status().unwrap().pid,
8172            before.pid,
8173            "changing a probe must apply live, not restart its process"
8174        );
8175        let (spec, mut health) = module.configuration().unwrap();
8176        health.failure_threshold = 2;
8177        module
8178            .update_configuration(spec, health, Some(10))
8179            .await
8180            .unwrap();
8181        status.store(503, std::sync::atomic::Ordering::SeqCst);
8182        timeout(Duration::from_secs(5), async {
8183            while module.status().unwrap().spawn_generation == before.spawn_generation {
8184                sleep(Duration::from_millis(5)).await;
8185            }
8186        })
8187        .await
8188        .expect("sustained HTTP 503 responses must trigger the health restart policy");
8189        module.drain().await.unwrap();
8190        server.abort();
8191    }
8192
8193    #[cfg(unix)]
8194    #[tokio::test]
8195    async fn http_health_replacement_recovers_after_one_restart() {
8196        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8197        let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
8198        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8199        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8200        let mut configured = http_fixture(&dir, &url, 2);
8201        configured.restart.max_restarts = 2;
8202        let module =
8203            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8204                .supervise_configured_with_health(
8205                    configured.module_spec(),
8206                    true,
8207                    configured.health,
8208                    configured.drain_timeout_ms,
8209                    configured.restart,
8210                )
8211                .unwrap();
8212        let first_generation = module.status().unwrap().spawn_generation;
8213        let observed = module.clone();
8214        let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
8215        let server = tokio::spawn(async move {
8216            let mut generation = first_generation;
8217            let mut probes = 0;
8218            loop {
8219                let (mut stream, _) = listener.accept().await.unwrap();
8220                let mut request = [0; 2048];
8221                assert!(stream.read(&mut request).await.unwrap() > 0);
8222                let current = observed.status().unwrap().spawn_generation;
8223                if current != generation {
8224                    generation = current;
8225                    probes = 0;
8226                }
8227                probes += 1;
8228                // The original process never recovers. Each replacement has one
8229                // transient miss, then answers successfully for its remaining life.
8230                let healthy = generation != first_generation && probes > 1;
8231                let code = if healthy { 200 } else { 503 };
8232                let response = format!(
8233                    "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
8234                );
8235                let _ = stream.write_all(response.as_bytes()).await;
8236                if healthy {
8237                    healthy_tx.send(()).await.unwrap();
8238                }
8239            }
8240        });
8241        timeout(Duration::from_secs(5), async {
8242            let mut healthy_probes = 0;
8243            loop {
8244                if module.state().unwrap() == ModuleState::Failed {
8245                    break;
8246                }
8247                while healthy_rx.try_recv().is_ok() {
8248                    healthy_probes += 1;
8249                }
8250                if healthy_probes >= 3
8251                    && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
8252                {
8253                    break;
8254                }
8255                sleep(Duration::from_millis(5)).await;
8256            }
8257        })
8258        .await
8259        .expect("the replacement must settle healthy without exhausting its restart budget");
8260        let status = module.status().unwrap();
8261        module.drain().await.unwrap();
8262        server.abort();
8263        assert_eq!(status.state, ModuleState::Running);
8264        assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
8265        assert_eq!(status.health.consecutive_failures, 0);
8266        assert_eq!(status.lifetime_restarts, 1);
8267        assert_eq!(status.restart_count, 1);
8268    }
8269
8270    #[cfg(unix)]
8271    #[tokio::test]
8272    async fn http_health_sustained_failure_exhausts_restart_budget() {
8273        use std::collections::BTreeMap;
8274        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8275        let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
8276        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8277        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8278        let mut configured = http_fixture(&dir, &url, 2);
8279        configured.restart.max_restarts = 2;
8280        let module =
8281            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8282                .supervise_configured_with_health(
8283                    configured.module_spec(),
8284                    true,
8285                    configured.health,
8286                    configured.drain_timeout_ms,
8287                    configured.restart,
8288                )
8289                .unwrap();
8290        let observed = module.clone();
8291        let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
8292        let served = probes.clone();
8293        let server = tokio::spawn(async move {
8294            loop {
8295                let (mut stream, _) = listener.accept().await.unwrap();
8296                let mut request = [0; 2048];
8297                assert!(stream.read(&mut request).await.unwrap() > 0);
8298                *served
8299                    .lock()
8300                    .unwrap()
8301                    .entry(observed.status().unwrap().spawn_generation)
8302                    .or_default() += 1;
8303                let _ = stream
8304                    .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
8305                    .await;
8306            }
8307        });
8308        timeout(Duration::from_secs(5), async {
8309            while module.state().unwrap() != ModuleState::Failed {
8310                sleep(Duration::from_millis(5)).await;
8311            }
8312        })
8313        .await
8314        .expect("sustained failures must still exhaust the health restart budget");
8315        let status = module.status().unwrap();
8316        server.abort();
8317        assert_eq!(status.restart_count, 2);
8318        assert_eq!(status.lifetime_restarts, 2);
8319        assert!(!status.process_alive);
8320        assert_eq!(
8321            probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
8322            vec![2, 2, 2]
8323        );
8324    }
8325
8326    #[cfg(unix)]
8327    #[tokio::test]
8328    async fn spawn_clears_the_previous_process_health() {
8329        let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
8330        let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
8331        configured.health.cadence = Duration::from_secs(3600);
8332        let module =
8333            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8334                .supervise_configured_with_health(
8335                    configured.module_spec(),
8336                    false,
8337                    configured.health,
8338                    configured.drain_timeout_ms,
8339                    configured.restart,
8340                )
8341                .unwrap();
8342        update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
8343            state.health.status = SupervisorHealthStatus::Unresponsive;
8344            state.health.consecutive_failures = 3;
8345            state.health.detail = Some("old process refused connections".into());
8346            state.health.metrics = Some(serde_json::json!({"old": true}));
8347            state.health.last_probe_ms = Some(123);
8348            state.health.last_action = Some("restart".into());
8349            state.health.last_action_ms = Some(124);
8350        })
8351        .unwrap();
8352        module.set_enabled(true).await.unwrap();
8353        let health = module.status().unwrap().health;
8354        module.drain().await.unwrap();
8355        assert_eq!(health.consecutive_failures, 0);
8356        assert_eq!(health.status, SupervisorHealthStatus::Unknown);
8357        assert_eq!(health.detail, None);
8358        assert_eq!(health.metrics, None);
8359        assert_eq!(health.last_probe_ms, None);
8360        assert_eq!(health.last_action.as_deref(), Some("restart"));
8361        assert_eq!(health.last_action_ms, Some(124));
8362    }
8363
8364    #[cfg(unix)]
8365    #[tokio::test]
8366    async fn http_health_no_listener_restarts_after_consecutive_failures() {
8367        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
8368        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8369        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
8370        drop(unused);
8371        let configured = http_fixture(&dir, &url, 2);
8372        let module =
8373            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8374                .supervise_configured_with_health(
8375                    configured.module_spec(),
8376                    true,
8377                    configured.health,
8378                    configured.drain_timeout_ms,
8379                    configured.restart,
8380                )
8381                .unwrap();
8382        let before = module.status().unwrap().spawn_generation;
8383        timeout(Duration::from_secs(5), async {
8384            loop {
8385                let status = module.status().unwrap();
8386                if status.spawn_generation > before {
8387                    assert!(status.lifetime_restarts > 0);
8388                    break;
8389                }
8390                sleep(Duration::from_millis(5)).await;
8391            }
8392        })
8393        .await
8394        .expect("sustained HTTP refusal must trigger the health restart policy");
8395        module.drain().await.unwrap();
8396    }
8397
8398    #[cfg(unix)]
8399    #[tokio::test]
8400    async fn http_health_timeout_honours_deadline() {
8401        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8402        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8403        let server = tokio::spawn(async move {
8404            let _held = listener.accept().await.unwrap();
8405            std::future::pending::<()>().await;
8406        });
8407        let error = timeout(
8408            Duration::from_secs(1),
8409            probe_http_health(&url, Duration::from_millis(10)),
8410        )
8411        .await
8412        .expect("the probe must enforce its own deadline")
8413        .unwrap_err();
8414        server.abort();
8415        assert!(error.to_string().contains("timed out"));
8416    }
8417
8418    #[cfg(unix)]
8419    #[tokio::test]
8420    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
8421        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8422        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
8423        let url = format!(
8424            "http://localhost:{}/healthz",
8425            listener.local_addr().unwrap().port()
8426        );
8427        let server = tokio::spawn(async move {
8428            let (mut stream, _) = listener.accept().await.unwrap();
8429            let mut request = [0u8; 2048];
8430            assert!(stream.read(&mut request).await.unwrap() > 0);
8431            stream
8432                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
8433                .await
8434                .unwrap();
8435        });
8436        // The deadline only bounds a hang. A probe that never tried the IPv6
8437        // address would be refused on 127.0.0.1 and fail at once, so a longer
8438        // deadline does not weaken the assertion; one second timed out under a
8439        // loaded parallel test run.
8440        assert_eq!(
8441            probe_http_health(&url, Duration::from_secs(10))
8442                .await
8443                .unwrap()
8444                .status,
8445            HealthStatus::Ok
8446        );
8447        server.await.unwrap();
8448    }
8449
8450    #[cfg(unix)]
8451    #[tokio::test]
8452    async fn http_health_timeout_keeps_partial_status_and_body() {
8453        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8454        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8455        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8456        let server = tokio::spawn(async move {
8457            let (mut stream, _) = listener.accept().await.unwrap();
8458            let mut request = [0u8; 2048];
8459            assert!(stream.read(&mut request).await.unwrap() > 0);
8460            stream
8461                .write_all(
8462                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
8463                )
8464                .await
8465                .unwrap();
8466            std::future::pending::<()>().await;
8467        });
8468        let error = probe_http_health(&url, Duration::from_secs(1))
8469            .await
8470            .unwrap_err()
8471            .to_string();
8472        server.abort();
8473        assert!(
8474            error.contains("timed out")
8475                && error.contains("503 Unavailable")
8476                && error.contains("partial diagnostic"),
8477            "{error}"
8478        );
8479    }
8480
8481    #[cfg(unix)]
8482    #[tokio::test]
8483    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
8484        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8485        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8486        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8487        let server = tokio::spawn(async move {
8488            let (mut stream, _) = listener.accept().await.unwrap();
8489            let mut request = [0u8; 2048];
8490            assert!(stream.read(&mut request).await.unwrap() > 0);
8491            let body = format!("{}not-in-diagnostic", "x".repeat(400));
8492            let response = format!(
8493                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
8494                body.len()
8495            );
8496            stream.write_all(response.as_bytes()).await.unwrap();
8497        });
8498        let error = probe_http_health(&url, Duration::from_secs(1))
8499            .await
8500            .unwrap_err()
8501            .to_string();
8502        server.await.unwrap();
8503        assert!(error.ends_with(&"x".repeat(200)), "{error}");
8504        assert_eq!(error.split(": ").last().unwrap().len(), 200);
8505        assert!(!error.contains("not-in-diagnostic"));
8506    }
8507
8508    #[cfg(unix)]
8509    #[tokio::test]
8510    async fn http_health_real_nats_server_monitoring() {
8511        if std::process::Command::new("nats-server")
8512            .arg("--version")
8513            .env("XDG_DATA_HOME", std::env::temp_dir())
8514            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
8515            .env("XDG_CONFIG_HOME", std::env::temp_dir())
8516            .output()
8517            .is_err()
8518        {
8519            eprintln!(
8520                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
8521            );
8522            return;
8523        }
8524        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
8525        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8526        let port = monitor.local_addr().unwrap().port();
8527        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8528        let client_port = client.local_addr().unwrap().port();
8529        let config = dir.join("server.conf");
8530        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
8531        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
8532        configured.program = PathBuf::from("nats-server");
8533        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
8534        drop(monitor);
8535        drop(client);
8536        let module =
8537            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8538                .supervise_configured_with_health(
8539                    configured.module_spec(),
8540                    true,
8541                    configured.health,
8542                    configured.drain_timeout_ms,
8543                    configured.restart,
8544                )
8545                .unwrap();
8546        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8547        module.drain().await.unwrap();
8548    }
8549
8550    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8551    async fn configuration_update_does_not_replace_captured_running_process_facts() {
8552        let supervisor =
8553            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8554        let snapshot = stale_process_snapshot(ModuleState::Running, true);
8555        let initial = ModuleSpec {
8556            script: None,
8557            module_id: "rescan-preserves-spawn-facts".to_string(),
8558            program: PathBuf::from("/spawned/module"),
8559            args: Vec::new(),
8560            env: Vec::new(),
8561            reserved: false,
8562            reserved_prefixes: Vec::new(),
8563            protocol: ModuleProtocol::Subc,
8564            overlap: Default::default(),
8565            carry_containment: false,
8566        };
8567        let module = supervisor.supervised_module(
8568            initial.clone(),
8569            supervisor.runtime_config(),
8570            snapshot,
8571            None,
8572        );
8573        let before = module.status().unwrap();
8574        let mut replacement = initial;
8575        replacement.program = PathBuf::from("/rescanned/replacement-module");
8576
8577        module
8578            .update_configuration(replacement, HealthConfig::default(), None)
8579            .await
8580            .unwrap();
8581
8582        let after = module.status().unwrap();
8583        assert_eq!(after.pid, before.pid);
8584        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
8585        assert_eq!(after.spawned_from, before.spawned_from);
8586        drop(module);
8587    }
8588}
8589
8590fn unix_ms_now() -> u64 {
8591    SystemTime::now()
8592        .duration_since(UNIX_EPOCH)
8593        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
8594        .unwrap_or(0)
8595}
8596
8597async fn supervise_loop(
8598    mut spec: ModuleSpec,
8599    mut runtime: SupervisorRuntimeConfig,
8600    registry: Arc<Registry>,
8601    process_liveness: Arc<SupervisorProcessLiveness>,
8602    snapshot: SharedSnapshot,
8603    mut child: Option<SupervisedChild>,
8604    mut commands: mpsc::Receiver<SupervisorCommand>,
8605) {
8606    let mut health_probe = HealthProbeRuntime::default();
8607    // Registry writes (including embedded callers) notify this module only.
8608    // Subscribe before the first refresh; watch retains changes that arrive
8609    // while commands or probes are running. No polling fallback is needed.
8610    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
8611        Ok(changes) => Some(changes),
8612        Err(err) => {
8613            // A poisoned registry cannot accept further writes, so it cannot
8614            // recover via a timer. Keep exit and command handling alive.
8615            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
8616            None
8617        }
8618    };
8619    // All restart backoffs run here, including health and operator requests.
8620    // While one is pending the loop serves commands, so disable or drain can
8621    // cancel the replacement without spawning a process just to stop it.
8622    let mut pending_respawn: Option<PendingRespawn> = None;
8623    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
8624    // before anything else so a stop that interrupted a swap runs at once.
8625    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
8626    let mut previous_instance_recheck = tokio::time::interval_at(
8627        Instant::now() + Duration::from_secs(5),
8628        Duration::from_secs(5),
8629    );
8630    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
8631    'supervision: loop {
8632        #[cfg(test)]
8633        {
8634            lock_snapshot(&snapshot).unwrap().actor_select = None;
8635        }
8636        #[cfg(target_os = "macos")]
8637        if let Some(active) = child.as_mut() {
8638            active.confirm_privacy_exec().await;
8639        }
8640        if let Some(scheduled) = runtime
8641            .scheduled_respawn
8642            .lock()
8643            .unwrap_or_else(|p| p.into_inner())
8644            .take()
8645        {
8646            runtime.recovery_is_daemon = scheduled.daemon_initiated;
8647            pending_respawn = Some(scheduled);
8648        }
8649        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
8650            pending_respawn = None;
8651            cancel_deferred_reload(
8652                &runtime,
8653                &spec.module_id,
8654                "respawn cancelled by a supervisor command",
8655            );
8656        }
8657        if child.is_none()
8658            && pending_respawn.is_none()
8659            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
8660        {
8661            cancel_deferred_reload(
8662                &runtime,
8663                &spec.module_id,
8664                "respawn cancelled before a replacement was spawned",
8665            );
8666            let mut stranded = false;
8667            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8668                state.respawn_pending = false;
8669                state.coalesced_restart_pending = false;
8670                if matches!(
8671                    state.state,
8672                    ModuleState::Restarting
8673                        | ModuleState::Starting
8674                        | ModuleState::Draining
8675                        | ModuleState::Unresponsive
8676                ) {
8677                    error!(module_id = %spec.module_id, state = ?state.state,
8678                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
8679                    stranded = true;
8680                }
8681            });
8682            if stranded {
8683                fail_snapshot(
8684                    &snapshot,
8685                    Some(&spec.module_id),
8686                    None,
8687                    &runtime.failure_context(runtime.recovery_is_daemon),
8688                );
8689            }
8690        }
8691        if child.is_none()
8692            && (runtime.child_roster.is_closed()
8693                || pending_respawn.is_none()
8694                || !lock_snapshot(&snapshot)
8695                    .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
8696        {
8697            containment::terminate(&runtime.child_roster.containment).await;
8698        }
8699        if let Some(command) = requeued.pop_front() {
8700            if !handle_supervisor_command(
8701                command,
8702                &mut spec,
8703                &mut runtime,
8704                &registry,
8705                &process_liveness,
8706                &snapshot,
8707                &mut child,
8708                &mut commands,
8709                &mut requeued,
8710            )
8711            .await
8712            {
8713                containment::terminate(&runtime.child_roster.containment).await;
8714                return;
8715            }
8716            if child.is_some() || !respawn_still_pending(&snapshot) {
8717                pending_respawn = None;
8718                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8719                    state.respawn_pending = false
8720                });
8721            }
8722            continue;
8723        }
8724        if child.is_some() {
8725            runtime.recovery_is_daemon = true;
8726            #[cfg(test)]
8727            {
8728                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
8729            }
8730            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
8731            let active = child.as_mut().expect("child checked above");
8732            if active.registered_since_spawn() {
8733                if let Some(delivery) = &runtime.config_delivery {
8734                    delivery.queue_registered(
8735                        &spec.module_id,
8736                        runtime.forwarding.as_ref(),
8737                        &registry,
8738                        active,
8739                    );
8740                }
8741            }
8742            if runtime.child_roster.is_closed() || !active.awaiting_registration() {
8743                active.registration_deadline = None;
8744            }
8745            let registration_deadline = active.registration_deadline;
8746            let no_checks = runtime.health.http.is_none()
8747                && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
8748                    || (health_probe.registered_connection.is_some() && !health_probe.advertised));
8749            let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
8750                state
8751                    .failed_retry
8752                    .uptime_reset_at(no_checks, runtime.restart_policy.window)
8753            });
8754            let wake_at = match (health_probe.wake_at(), uptime_reset) {
8755                (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
8756                (probe, None) => probe,
8757            };
8758            let wake_at = match (wake_at, registration_deadline) {
8759                (Some(wake), Some(register)) => Some(wake.min(register)),
8760                (wake, register) => wake.or(register),
8761            };
8762            #[cfg(test)]
8763            let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
8764            let probe_sleep = supervisor_sleep(wake_at);
8765            tokio::pin!(probe_sleep);
8766            // A `protocol: "none"` child (a plain process that never registers
8767            // over the subc wire, such as nats-server) with no HTTP health
8768            // check has nothing to probe, so it waits only for its exit or a
8769            // supervisor command. A rescan that adds an HTTP check is a command.
8770            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
8771            #[cfg(test)]
8772            {
8773                let registration_pending = wire_child
8774                    && registration_changes
8775                        .as_ref()
8776                        .is_some_and(|changes| changes.has_changed().unwrap());
8777                let mut state = lock_snapshot(&snapshot).unwrap();
8778                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
8779                    generation: state.spawn_generation,
8780                    turn: state.actor_turns,
8781                    registered_connection: health_probe.registered_connection,
8782                    next_probe_at: health_probe.next_probe_at,
8783                    wake_after,
8784                });
8785            }
8786            let registration = active
8787                .awaiting_registration()
8788                .then(|| active.registration_spawn.clone())
8789                .flatten();
8790            #[cfg(all(test, target_os = "linux"))]
8791            let registration_before_exit = !runtime.child_roster.admission_event_for_test(
8792                &spec.module_id,
8793                "observe_registration_before_exit",
8794                Some(active.pid),
8795            ) && registration.is_some();
8796            #[cfg(not(all(test, target_os = "linux")))]
8797            let registration_before_exit = false;
8798            #[cfg(all(test, target_os = "linux"))]
8799            let registration_child_is_running = |child: &mut SupervisedChild| {
8800                // Model the interval after SIGKILL is accepted but before the
8801                // child becomes waitable, without delaying the actual kill.
8802                registration_before_exit || registration_child_is_running(child)
8803            };
8804            let registration_failed = async {
8805                match &registration {
8806                    Some(spawn) => spawn.wait_registration_failure().await,
8807                    None => std::future::pending().await,
8808                }
8809            };
8810            let active_child = child.as_mut().expect("child checked above");
8811            tokio::select! {
8812                _ = registration_failed => {
8813                    if !active_child.registered_since_spawn() && registration_child_is_running(active_child) {
8814                        active_child.registration_deadline = None;
8815                        active_child.spawn_failure = Some("registration_channel_closed".into());
8816                        if let Err(error) = stop_unregistered_child(active_child).await {
8817                            fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8818                                &runtime.spawn_events, format!("registration_channel_closed: {error}"),
8819                                &runtime.failure_context(true));
8820                            child = None;
8821                        }
8822                    }
8823                    continue;
8824                }
8825                wait_result = active_child.wait(), if !registration_before_exit => {
8826                    if let Ok(mut state) = lock_snapshot(&snapshot) {
8827                        state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
8828                    }
8829                    #[cfg(test)]
8830                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8831                    // Every arm below that gives up on the CHILD must keep the
8832                    // supervision task itself alive (child = None, loop
8833                    // continues into command-serving mode). Returning here
8834                    // closes the command channel, which makes the module
8835                    // permanently unrestartable in-band: a clean child exit
8836                    // of an enabled module once wedged the fleet this way
8837                    // ('supervisor command channel is closed') and required a
8838                    // full daemon restart to recover.
8839                    let exit_report = match wait_result {
8840                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
8841                        Err(err) => {
8842                            active_child.drain_stderr(&spec.module_id).await;
8843                            fail_after_wait_error(
8844                                &spec.module_id,
8845                                &snapshot,
8846                                &runtime.terminal_ring,
8847                                &runtime.spawn_events,
8848                                format!("failed to wait for supervised module: {err}"),
8849                            &runtime.failure_context(true),
8850                            );
8851                            untrack_if_registration_released(
8852                                &process_liveness,
8853                                &registry,
8854                                &spec.module_id,
8855                                &snapshot,
8856                            );
8857                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
8858                            child = None;
8859                            continue;
8860                        }
8861                    };
8862                    active_child.drain_stderr(&spec.module_id).await;
8863
8864                    let delivery=runtime.config_delivery.clone();
8865                    let exited_spawn=active_child.spawned_at;
8866                    let spawn=active_child.config_delivery.clone();
8867                    let refusal=active_child.spawn_failure.as_deref().filter(|detail|detail.starts_with("program_refused:") && !active_child.live_program_binding.as_ref().is_some_and(|binding|binding.is_refused())).zip(active_child.admission_base.clone()).map(|(detail,base)|(base,subc_config::raw_digest(detail.as_bytes())));
8868                    let module_id=spec.module_id.clone();
8869                    // Publication may own the store mutex for local durable I/O.
8870                    // Keep commands selectable until the child's exit is recorded;
8871                    // no respawn disposition is chosen before that observation.
8872                    let observed=async move {
8873                        if let Some(delivery)=delivery {
8874                            delivery.0.on_blocking(move |owner| {
8875                                if let Some(spawn)=spawn { owner.delivery_child_exited(&spawn)?; }
8876                                if let Some((base,identity))=refusal {owner.record_program_refusal(&module_id,&base,identity)?;}
8877                                Ok(())
8878                            }).await
8879                        } else {Ok(())}
8880                    };
8881                    tokio::pin!(observed);
8882                    loop {
8883                        tokio::select! {
8884                            result=&mut observed=>{
8885                                if let Err(error)=result {warn!(module_id=%spec.module_id,%error,"configuration exit observation failed");}
8886                                break;
8887                            }
8888                            command=commands.recv()=>{
8889                                let Some(command)=command else {containment::terminate(&runtime.child_roster.containment).await;return;};
8890                                let deferred=||SuperviseError::ConfigDeliveryDeferred {module_id:spec.module_id.clone()};
8891                                let command=match command {
8892                                    SupervisorCommand::Restart {reply,..}|SupervisorCommand::Reload {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8893                                    SupervisorCommand::Swap {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8894                                    command @ (SupervisorCommand::ConfigApplyRestart {..}|SupervisorCommand::ConfigPolicyProbe {..}|SupervisorCommand::ConfigDeliveryFailed {..})=>{requeued.push_back(command);continue;}
8895                                    command=>command,
8896                                };
8897                                if !handle_supervisor_command(command,&mut spec,&mut runtime,&registry,&process_liveness,&snapshot,&mut child,&mut commands,&mut requeued).await {
8898                                    containment::terminate(&runtime.child_roster.containment).await;return;
8899                                }
8900                                if child.as_ref().is_none_or(|current|current.spawned_at!=exited_spawn) {continue 'supervision;}
8901                            }
8902                        }
8903                    }
8904                    let active_child=child.as_mut().expect("exit observation retained its child");
8905                    // An unavailable image can race the program's own exit.
8906                    // Only the admission kill is budget-free; another exit must
8907                    // retain its ordinary crash/clean-exit accounting.
8908                    let admission_kill = admission_exit::is_admission_kill(active_child.admission_kill_sent, &exit_report);
8909                    let deferred_spawn=admission_kill && active_child.spawn_failure.as_deref().is_some_and(|detail|detail.starts_with("config_delivery_deferred:"));
8910                    let refused_program = active_child.live_program_binding.as_ref().is_some_and(|binding| binding.is_refused());
8911                    let exit_held = !deferred_spawn && (refused_program || runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.exit_is_held(&spec.module_id)));
8912                    let parked = exit_held && (!refused_program || active_child.admission_base.is_some())
8913                        && try_park_config_attempt(&spec, &runtime, &snapshot, &runtime.failure_context(true),
8914                            if refused_program { active_child.admission_base.as_ref() } else { None })
8915                            .unwrap_or_else(|error| { warn!(module_id=%spec.module_id, %error, "exit hold could not be recorded; using ordinary exit accounting"); false });
8916                    let next = if deferred_spawn {
8917                        record_terminal_with_detail(&spec.module_id,&runtime.terminal_ring,&runtime.spawn_events,&exit_report,TerminalDisposition::Restarting,active_child.spawn_failure.clone());
8918                        let _=update_snapshot(&snapshot,Some(&spec.module_id),|state| {
8919                            state.state=ModuleState::Restarting;
8920                            clear_current_process_facts(state);
8921                            state.last_exit=Some(exit_report.clone());
8922                        });
8923                        NextAction::Restart {schedule:None}
8924                    } else if parked {
8925                        record_terminal_with_detail(&spec.module_id, &runtime.terminal_ring, &runtime.spawn_events, &exit_report, TerminalDisposition::Stopped, active_child.spawn_failure.clone().or_else(|| Some("config_gated".into())));
8926                        NextAction::Stop { registration_released: registry.get_module(&spec.module_id).ok().flatten().is_none() }
8927                    } else { on_child_exit(
8928                        &spec,
8929                        runtime.restart_policy,
8930                        &registry,
8931                        &snapshot,
8932                        &runtime.terminal_ring,
8933                        &runtime.spawn_events,
8934                        &runtime.child_roster,
8935                        exit_report,
8936                        true,
8937                    ).await };
8938                    runtime.start_requester = None;
8939                    runtime.pending_activity_restart = false;
8940                    // The exit is recorded, so a daemon shutdown may stop
8941                    // waiting for this child (see `SupervisedChild::wait`).
8942                    let carry = matches!(next, NextAction::Restart { .. })
8943                        && active_child.spawn_failure.is_none()
8944                        && !runtime.child_roster.is_closed();
8945                    if let Err(error) = active_child.finish_exit(carry).await {
8946                        containment::terminate(&runtime.child_roster.containment).await;
8947                        fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8948                            &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
8949                            &runtime.failure_context(true));
8950                        child = None;
8951                        continue;
8952                    }
8953                    match next {
8954                        NextAction::Stop { registration_released } => {
8955                            if registration_released {
8956                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
8957                            }
8958                            child = None;
8959                        }
8960                        NextAction::Restart { schedule } => {
8961                            let delay = if deferred_spawn {CONFIG_ADMISSION_RETRY_DELAY} else {schedule.map_or(
8962                                runtime.restart_policy.delay_for_restart(0),
8963                                |schedule| schedule.delay,
8964                            )};
8965                            if let Some(schedule) = schedule {
8966                                log_crash_respawn(&spec.module_id, schedule);
8967                            }
8968                            // The exited child is fully recorded at this point,
8969                            // so release it and count the backoff down in the
8970                            // command-serving branch below rather than sleeping
8971                            // here: commands cannot be received from inside this
8972                            // select arm, and an operator disable or drain that
8973                            // arrives during the backoff must cancel the pending
8974                            // respawn instead of waiting for it to spawn first.
8975                            child = None;
8976                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true, approval: None });
8977                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
8978                        }
8979                    }
8980                }
8981                command = commands.recv() => {
8982                    #[cfg(test)]
8983                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8984                    let Some(command) = command else {
8985                        containment::terminate(&runtime.child_roster.containment).await;
8986                        return;
8987                    };
8988                    if !handle_supervisor_command(
8989                        command,
8990                        &mut spec,
8991                        &mut runtime,
8992                        &registry,
8993                        &process_liveness,
8994                        &snapshot,
8995                        &mut child,
8996                        &mut commands,
8997                        &mut requeued,
8998                    ).await {
8999                        containment::terminate(&runtime.child_roster.containment).await;
9000                        return;
9001                    }
9002                }
9003                _ = async {
9004                    match registration_changes.as_mut() {
9005                        Some(changes) => { let _ = changes.changed().await; }
9006                        None => std::future::pending().await,
9007                    }
9008                }, if wire_child => {
9009                    #[cfg(test)]
9010                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
9011                }
9012                _ = &mut probe_sleep => {
9013                    #[cfg(test)]
9014                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
9015                    if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
9016                        // Recheck after the wake: HELLO, exit or shutdown may
9017                        // have become ready in the same executor tick.
9018                        active_child.registration_deadline = None;
9019                        if !runtime.child_roster.is_closed()
9020                            && !active_child.registered_since_spawn()
9021                            && registration_child_is_running(active_child)
9022                        {
9023                            warn!(module_id = %spec.module_id, pid = active_child.pid,
9024                                wait_secs = runtime.register_timeout.as_secs(),
9025                                "supervised module registration_timeout: first HELLO deadline expired; stopping child");
9026                            active_child.spawn_failure = Some("registration_timeout".into());
9027                            if let Err(error) = stop_unregistered_child(active_child).await {
9028                                error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
9029                                fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
9030                                    &runtime.spawn_events, format!("registration_timeout: {error}"),
9031                                    &runtime.failure_context(true));
9032                                child = None;
9033                            }
9034                            // The ordinary exit handler records the child's exit,
9035                            // spends crash budget, and schedules recovery.
9036                            continue;
9037                        }
9038                    }
9039                    if health_probe.due() {
9040                        run_health_probe_cycle(
9041                            &spec,
9042                            &runtime,
9043                            &registry,
9044                            &process_liveness,
9045                            &snapshot,
9046                            &mut child,
9047                        ).await;
9048                        if child.is_none() {
9049                            runtime.start_requester = None;
9050                            runtime.pending_activity_restart = false;
9051                        }
9052                        if child.is_some() {
9053                            health_probe.schedule_next(&spec, runtime.health.cadence);
9054                        }
9055                    }
9056                }
9057            }
9058        } else if let Some(pending) = pending_respawn.clone() {
9059            tokio::select! {
9060                _ = runtime.child_roster.closed() => {
9061                    pending_respawn = None;
9062                    let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9063                        state.state = ModuleState::Stopped;
9064                        state.respawn_pending = false;
9065                    });
9066                    containment::terminate(&runtime.child_roster.containment).await;
9067                }
9068                _ = async {
9069                    sleep_until(pending.deadline).await;
9070                    if let Some(delivery) = &runtime.config_delivery {
9071                        delivery.unfenced(&spec.module_id).await;
9072                    }
9073                } => {
9074                    pending_respawn = None;
9075                    let failure = pending.failure_context(&runtime);
9076                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
9077                    // A command handled below while the backoff elapsed may
9078                    // have stopped the module; never respawn past an operator's
9079                    // disable or drain.
9080                    if !respawn_still_pending(&snapshot) {
9081                        continue;
9082                    }
9083                    // The daemon began shutting down during the backoff: the
9084                    // spawn would be refused anyway, and refusing it here
9085                    // leaves the module stopped instead of reporting a
9086                    // failed restart.
9087                    if runtime.child_roster.is_closed() {
9088                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9089                            state.state = ModuleState::Stopped;
9090                        });
9091                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
9092                        continue;
9093                    }
9094                    if let Err(err) = release_dead_registration(
9095                        &registry,
9096                        runtime.forwarding.as_deref(),
9097                        &snapshot,
9098                        &spec.module_id,
9099                        &failure,
9100                    ).await {
9101                        record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
9102                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
9103                        continue;
9104                    }
9105
9106                    if matches!(pending.kind, RespawnKind::Reload) {
9107                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
9108                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
9109                        if matches!(result, Err(SuperviseError::ConfigDeliveryDeferred { .. })) {
9110                            *runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()) = reply;
9111                            continue;
9112                        }
9113                        if result.is_ok() && runtime.pending_activity_restart {
9114                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
9115                        }
9116                        runtime.pending_activity_restart = false;
9117                        if let Some(reply) = reply { let _ = reply.send(result); }
9118                        continue;
9119                    }
9120                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
9121                    match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
9122                        Ok(next_child) => {
9123                            child = Some(next_child);
9124                            if runtime.pending_activity_restart {
9125                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
9126                                runtime.pending_activity_restart = false;
9127                            }
9128                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
9129                        }
9130                        Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9131                            pending_respawn = Some(PendingRespawn { deadline:Instant::now()+CONFIG_ADMISSION_RETRY_DELAY, ..pending });
9132                            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
9133                        }
9134                        Err(SuperviseError::ConfigHeld { .. }) => {
9135                            let _ = hold_config_spawn(&spec, &runtime, &snapshot, &failure);
9136                            runtime.pending_activity_restart = false;
9137                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9138                        }
9139                        Err(err) => {
9140                            fail_snapshot(&snapshot, Some(&spec.module_id), None, &failure);
9141                            record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
9142                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9143                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
9144                        }
9145                    }
9146                }
9147                command = commands.recv() => {
9148                    let Some(command) = command else {
9149                        containment::terminate(&runtime.child_roster.containment).await;
9150                        return;
9151                    };
9152                    if !handle_supervisor_command(
9153                        command,
9154                        &mut spec,
9155                        &mut runtime,
9156                        &registry,
9157                        &process_liveness,
9158                        &snapshot,
9159                        &mut child,
9160                        &mut commands,
9161                        &mut requeued,
9162                    ).await {
9163                        containment::terminate(&runtime.child_roster.containment).await;
9164                        return;
9165                    }
9166                    // Reconcile the pending respawn with what the command did:
9167                    // a start may already have spawned a fresh child,
9168                    // while a disable or drain moved the snapshot out of the
9169                    // state the respawn was counting down from.
9170                    if child.is_some() || !respawn_still_pending(&snapshot) {
9171                        pending_respawn = None;
9172                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
9173                    }
9174                }
9175            }
9176        } else {
9177            let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
9178                (state.enabled && state.state == ModuleState::Failed)
9179                    .then_some(state.failed_retry.pending)
9180                    .flatten()
9181            });
9182            let command = if lock_snapshot(&snapshot)
9183                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
9184            {
9185                tokio::select! {
9186                    command = commands.recv() => command,
9187                    _ = previous_instance_recheck.tick() => {
9188                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
9189                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
9190                            if enabled && !runtime.child_roster.is_closed() {
9191                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
9192                                match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
9193                                    Ok(next_child) => child = Some(next_child),
9194                                    Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9195                                        let _ = defer_config_spawn(&spec, &runtime, &snapshot, RespawnKind::Spawn, &runtime.failure_context(true));
9196                                    }
9197                                    Err(SuperviseError::ConfigHeld { .. }) => {
9198                                        let _ = hold_config_spawn(&spec, &runtime, &snapshot, &runtime.failure_context(true));
9199                                    }
9200                                    Err(error) => {
9201                                        fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
9202                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
9203                                    }
9204                                }
9205                            } else {
9206                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9207                                    state.waiting_for_previous_pid = None;
9208                                    state.waiting_for_previous_detail = None;
9209                                });
9210                            }
9211                        }
9212                        continue;
9213                    }
9214                }
9215            } else {
9216                tokio::select! {
9217                    biased;
9218                    command = commands.recv() => command,
9219                    _ = async {
9220                        match slow_retry {
9221                            Some(retry) => sleep_until(retry.deadline).await,
9222                            None => std::future::pending().await,
9223                        }
9224                    } => {
9225                        let retry = slow_retry.expect("only a scheduled slow retry can wake");
9226                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9227                            state.failed_retry.pending = None;
9228                        });
9229                        if runtime.child_roster.is_closed() {
9230                            continue;
9231                        }
9232                        info!(module_id = %spec.module_id, attempt = retry.attempt,
9233                            delay_secs = retry.delay.as_secs(), "starting failed module retry");
9234                        if let Err(error) = set_child_enabled(
9235                            &spec, &runtime, &registry, &process_liveness, &snapshot, &mut child, true,
9236                            &runtime.failure_context(true),
9237                        ).await {
9238                            error!(module_id = %spec.module_id, %error, "failed module retry could not start");
9239                        }
9240                        continue;
9241                    }
9242                }
9243            };
9244            let Some(command) = command else {
9245                containment::terminate(&runtime.child_roster.containment).await;
9246                return;
9247            };
9248            if !handle_supervisor_command(
9249                command,
9250                &mut spec,
9251                &mut runtime,
9252                &registry,
9253                &process_liveness,
9254                &snapshot,
9255                &mut child,
9256                &mut commands,
9257                &mut requeued,
9258            )
9259            .await
9260            {
9261                containment::terminate(&runtime.child_roster.containment).await;
9262                return;
9263            }
9264        }
9265    }
9266}
9267
9268// Preserve the deadline until the future is polled. Starting a relative sleep
9269// after the executor yields would restart part of the countdown.
9270async fn supervisor_sleep(wake_at: Option<Instant>) {
9271    match wake_at {
9272        Some(deadline) => sleep_until(deadline).await,
9273        None => std::future::pending().await,
9274    }
9275}
9276
9277fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
9278    info!(
9279        module_id,
9280        restart_in_window = schedule.restart_in_window,
9281        delay_ms = schedule.delay.as_millis() as u64,
9282        "respawning after crash"
9283    );
9284}
9285
9286/// Whether the respawn a backoff was counting down to is still wanted. A
9287/// disable or drain handled while the backoff elapsed moves the snapshot out
9288/// of `Restarting`, and the operator's stop must win over the pending respawn,
9289/// so every sleep-then-spawn path re-validates against the live snapshot
9290/// instead of assuming the state it left behind still holds.
9291fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
9292    matches!(
9293        lock_snapshot(snapshot),
9294        Ok(state) if state.enabled && state.state == ModuleState::Restarting
9295    )
9296}
9297
9298enum NextAction {
9299    Stop {
9300        registration_released: bool,
9301    },
9302    Restart {
9303        schedule: Option<CrashRestartSchedule>,
9304    },
9305}
9306
9307#[allow(clippy::too_many_arguments)]
9308async fn handle_supervisor_command(
9309    command: SupervisorCommand,
9310    spec: &mut ModuleSpec,
9311    runtime: &mut SupervisorRuntimeConfig,
9312    registry: &Arc<Registry>,
9313    process_liveness: &SupervisorProcessLiveness,
9314    snapshot: &SharedSnapshot,
9315    child: &mut Option<SupervisedChild>,
9316    commands: &mut mpsc::Receiver<SupervisorCommand>,
9317    requeued: &mut VecDeque<SupervisorCommand>,
9318) -> bool {
9319    if let SupervisorCommand::SetEnabled {
9320        enabled: true,
9321        reset_budget: false,
9322        reply,
9323        ..
9324    } = command
9325    {
9326        let parked = lock_snapshot(snapshot).ok().and_then(|state| {
9327            (state.config_hold_parked
9328                && state.enabled
9329                && !state.process_alive
9330                && child.is_none()
9331                && state.state == ModuleState::Stopped)
9332                .then(|| {
9333                    (
9334                        state.config_hold_daemon_initiated,
9335                        state.config_hold_approval.clone(),
9336                    )
9337                })
9338        });
9339        // Clearing a configuration-delivery block must not restart a child that
9340        // exited on its own or exhausted its crash budget. Resume only an actor
9341        // stopped by a configuration hold; otherwise leave its slow retry alone.
9342        let result = if let Some((daemon_initiated, approval)) = parked {
9343            runtime.recovery_is_daemon = daemon_initiated;
9344            let failure = FailureContext {
9345                approval,
9346                ..runtime.failure_context(daemon_initiated)
9347            };
9348            let result = set_child_enabled_with_budget_policy(
9349                spec,
9350                runtime,
9351                registry,
9352                process_liveness,
9353                snapshot,
9354                child,
9355                true,
9356                &failure,
9357                false,
9358            )
9359            .await;
9360            if let Err(error) = &result {
9361                record_approved_attempt_failure(spec, snapshot, &failure, error);
9362            }
9363            result
9364        } else {
9365            Ok(false)
9366        };
9367        let _ = reply.send(result);
9368        return true;
9369    }
9370    let blocked = runtime
9371        .config_delivery
9372        .as_ref()
9373        .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
9374        .filter(|error| error.code != "config_delivery_deferred")
9375        .map(|error| {
9376            config_delivery::gate_hold_error(spec, error, runtime.config_delivery.as_ref())
9377        })
9378        .or_else(|| recovery_block_error(spec, &runtime.child_roster))
9379        .or_else(|| {
9380            runtime
9381                .child_roster
9382                .previous_instance(&spec.module_id)
9383                .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
9384        });
9385    let command = if let Some(error) = blocked {
9386        match command {
9387            SupervisorCommand::Restart { reply, .. } | SupervisorCommand::Reload { reply, .. } => {
9388                let _ = reply.send(Err(error));
9389                return true;
9390            }
9391            SupervisorCommand::Swap { reply, .. } => {
9392                let _ = reply.send(Err(error));
9393                return true;
9394            }
9395            SupervisorCommand::ConfigApplyRestart { .. } => return true,
9396            SupervisorCommand::SetEnabled {
9397                enabled: true,
9398                requester,
9399                reply,
9400                ..
9401            } => {
9402                if matches!(error, SuperviseError::ConfigHeld { .. })
9403                    && lock_snapshot(snapshot).is_ok_and(|state| !state.enabled)
9404                    && runtime
9405                        .config_delivery
9406                        .as_ref()
9407                        .is_some_and(|delivery| delivery.0.can_stage_unstamp_enable())
9408                {
9409                    // Stage only the pending unstamp's desired state. A stamped
9410                    // unapproved actor stays refused; admission release starts
9411                    // an unstamped actor only after reconciliation completes.
9412                    let mut changed = false;
9413                    let result = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9414                        changed = !state.enabled;
9415                        state.enabled = true;
9416                        if child.is_none() && state.waiting_for_previous_pid.is_none() {
9417                            state.state = ModuleState::Starting;
9418                            state.failed_retry.cancel();
9419                        }
9420                    })
9421                    .and_then(|_| {
9422                        if child.is_none()
9423                            && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none()
9424                        {
9425                            hold_config_spawn(
9426                                spec,
9427                                runtime,
9428                                snapshot,
9429                                &runtime.failure_context(false),
9430                            )?;
9431                        }
9432                        Ok(changed)
9433                    });
9434                    if result.is_ok() && changed {
9435                        runtime.start_requester = requester;
9436                    }
9437                    let _ = reply.send(result);
9438                    return true;
9439                }
9440                let _ = reply.send(Err(error));
9441                return true;
9442            }
9443            other => other,
9444        }
9445    } else {
9446        command
9447    };
9448    // A refused start or restart leaves the Failed actor's existing recovery
9449    // deadline intact. Cancel only after the command passes admission.
9450    if matches!(
9451        command,
9452        SupervisorCommand::Drain { .. }
9453            | SupervisorCommand::Retire { .. }
9454            | SupervisorCommand::SetEnabled { .. }
9455            | SupervisorCommand::Restart { .. }
9456    ) {
9457        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9458            state.failed_retry.cancel();
9459        });
9460    }
9461    match command {
9462        SupervisorCommand::ApprovedConfiguration { approved, reply } => {
9463            let result = async {
9464                let next = approved.module_entry(&spec.module_id).map_err(|error| {
9465                    SuperviseError::InvalidSpec { reason: error.to_string() }
9466                })?;
9467                let retry_approved_failure = {
9468                    let mut state = lock_snapshot(snapshot)?;
9469                    if let Some((generation, detail)) = &state.approved_start_failure {
9470                        if *generation == approved.identity() && next.enabled {
9471                            return Err(SuperviseError::Spawn {
9472                                program: spec.program.clone(),
9473                                source: std::io::Error::other(detail.clone()),
9474                                cgroup_path: None,
9475                            });
9476                        }
9477                    }
9478                    let retry = state.approved_start_failure.is_some();
9479                    state.approved_start_failure = None;
9480                    retry
9481                };
9482                stage_supervisor_configuration(
9483                    spec, runtime, snapshot, next.module_spec(), next.health,
9484                    next.drain_timeout_ms,
9485                );
9486                runtime.restart_policy = next.restart;
9487                runtime.register_timeout = next.register_timeout;
9488                if let Some(active) = child.as_mut()
9489                    .filter(|child| child.registration_deadline.is_some())
9490                {
9491                    active.registration_deadline = Some(active.spawned_at + next.register_timeout);
9492                }
9493                let enabled = lock_snapshot(snapshot)?.enabled;
9494                // An approval retries a changed failed entry or its own earlier
9495                // failed start, not an unrelated crash-budget failure. For an
9496                // unchanged entry, reconciled means the approved configuration
9497                // is effective; it does not claim the process has recovered.
9498                // A terminal gate stop is different: a new approval owns its
9499                // recovery even when the daemon entry itself did not change.
9500                let retry_gate_stopped = lock_snapshot(snapshot)?
9501                    .needs_approved_gate_restart(&approved.identity());
9502                let retry_failed = enabled && child.is_none()
9503                    && ((lock_snapshot(snapshot)?.state == ModuleState::Failed
9504                        && (retry_approved_failure || approved.changes_entry(&spec.module_id)))
9505                        || retry_gate_stopped);
9506                if retry_failed {
9507                    // This approval owns a new start, so the failed attempt's
9508                    // daemon retry must not survive either result of that start.
9509                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
9510                        state.failed_retry.cancel();
9511                        if retry_gate_stopped {
9512                            state.stop_reason = None;
9513                            state.config_exit_failed_approval = None;
9514                        }
9515                    })?;
9516                }
9517                if enabled != next.enabled || retry_failed {
9518                    let failure = FailureContext {
9519                        approval: next.enabled.then(|| approved.identity()),
9520                        ..runtime.failure_context(false)
9521                    };
9522                    let held = next.enabled && runtime.config_delivery.as_ref()
9523                        .is_some_and(|delivery| delivery.0.has_recorded_admission_hold(&spec.module_id)
9524                            || delivery.0.check_module_hold(&spec.module_id).is_err_and(|error| error.code == "config_delivery_deferred"));
9525                    if held {
9526                        // Preserve module-delivery failure/recovery holds. The
9527                        // admission release starts the actor when its own hold ends.
9528                        reset_restart_count(snapshot, &spec.module_id)?;
9529                        update_snapshot(snapshot, Some(&spec.module_id), |state| {
9530                            state.enabled = true;
9531                            if child.is_none() && state.waiting_for_previous_pid.is_none() {
9532                                state.state = ModuleState::Starting;
9533                            }
9534                        })?;
9535                        if child.is_none() && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none() {
9536                            hold_config_spawn(spec, runtime, snapshot, &failure)?;
9537                        }
9538                    } else {
9539                        match set_child_enabled_with_budget_policy(
9540                            spec, runtime, registry, process_liveness, snapshot, child,
9541                            next.enabled, &failure, true,
9542                        ).await {
9543                            Ok(_) => {},
9544                            Err(SuperviseError::ConfigHeld { .. }) if next.enabled => {},
9545                            Err(error) => return Err(error),
9546                        }
9547                        if !next.enabled {
9548                            containment::terminate(&runtime.child_roster.containment).await;
9549                        }
9550                    }
9551                }
9552                if (child.as_ref().is_some_and(|child| child.registration_spawn.is_some() && child.admission_base.is_none())
9553                        || (child.is_none() && lock_snapshot(snapshot)?.state == ModuleState::Restarting
9554                            && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.stamp_channel_cutover_pending())))
9555                    && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.module_is_gated(&spec.module_id))
9556                {
9557                    let failure = FailureContext {
9558                        approval: Some(approved.identity()),
9559                        ..runtime.failure_context(false)
9560                    };
9561                    // A stamp cannot adopt an unverified legacy process, even
9562                    // when its approved replacement no longer speaks subc.
9563                    // Stopping that process spends no crash restart budget.
9564                    if child.is_some() {
9565                        restart_child_with_budget_policy(spec, runtime, registry,
9566                            process_liveness, snapshot, child, runtime.drain_timeout, false).await?;
9567                    }
9568                    // The stamp is already committed. A deferred or held launch
9569                    // belongs to the normal retry/hold loop, not a failed apply.
9570                    update_snapshot(snapshot, Some(&spec.module_id), |state| state.respawn_pending = false)?;
9571                    match spawn_and_mark_running_async(spec, runtime, snapshot).await {
9572                        Ok(replacement) => *child = Some(replacement),
9573                        Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9574                            defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, &failure)?;
9575                        }
9576                        Err(SuperviseError::ConfigHeld { .. }) => hold_config_spawn(spec, runtime, snapshot, &failure)?,
9577                        Err(error) => {
9578                            fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
9579                            return Err(error);
9580                        }
9581                    }
9582                }
9583                info!(module_id=%spec.module_id, config_generation=approved.generation(), "approved daemon entry reconciled");
9584                Ok(())
9585            }
9586            .await;
9587            if let Err(error) = &result {
9588                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9589                    if state.enabled
9590                        && state.state == ModuleState::Failed
9591                        && state
9592                            .approved_start_failure
9593                            .as_ref()
9594                            .is_none_or(|(generation, _)| *generation != approved.identity())
9595                    {
9596                        state.approved_start_failure =
9597                            Some((approved.identity(), error.to_string()));
9598                    }
9599                });
9600            }
9601            let _ = reply.send(result);
9602            true
9603        }
9604        SupervisorCommand::ConfigPolicyProbe {
9605            connection,
9606            generation,
9607            attempt,
9608        } => {
9609            let Some(delivery) = &runtime.config_delivery else {
9610                return true;
9611            };
9612            let Ok(gate) = delivery.0.module_gate(&spec.module_id) else {
9613                return true;
9614            };
9615            let guard = gate.lock().await;
9616            let pending = delivery
9617                .0
9618                .on_blocking(move |owner| {
9619                    owner.policy_probe_pending(connection, generation, attempt)
9620                })
9621                .await
9622                .unwrap_or(false)
9623                && child.is_some()
9624                && registry
9625                    .get_module(&spec.module_id)
9626                    .ok()
9627                    .flatten()
9628                    .is_some_and(|registration| registration.connection_id == connection);
9629            drop(guard);
9630            if pending {
9631                run_health_probe_cycle(spec, runtime, registry, process_liveness, snapshot, child)
9632                    .await;
9633            }
9634            true
9635        }
9636        SupervisorCommand::ConfigApplyRestart {
9637            generation,
9638            supersedes_swap,
9639        } => {
9640            let Some(owner) = runtime
9641                .config_delivery
9642                .as_ref()
9643                .map(|delivery| delivery.0.clone())
9644            else {
9645                return true;
9646            };
9647            let Ok(gate) = owner.module_gate(&spec.module_id) else {
9648                return true;
9649            };
9650            let _guard = gate.lock().await;
9651            let spawn = child
9652                .as_ref()
9653                .and_then(|child| child.config_delivery.clone());
9654            let module = spec.module_id.clone();
9655            let current = owner
9656                .on_blocking(move |owner| {
9657                    Ok((supersedes_swap
9658                        || owner
9659                            .pending_config_restarts()?
9660                            .contains(&(module.clone(), generation)))
9661                        && owner.requires_fresher_spawn(&module, spawn.as_ref())?
9662                            == Some(generation))
9663                })
9664                .await
9665                .unwrap_or(false);
9666            if !current
9667                || !lock_snapshot(snapshot).is_ok_and(|state| {
9668                    state.enabled && !state.respawn_pending && state.process_alive
9669                })
9670            {
9671                return true;
9672            }
9673            drop(_guard);
9674            info!(module_id=%spec.module_id, config_generation=generation, "approved configuration restart preserves the crash budget");
9675            if let Err(error) = restart_child_with_budget_policy(
9676                spec,
9677                runtime,
9678                registry,
9679                process_liveness,
9680                snapshot,
9681                child,
9682                runtime.drain_timeout,
9683                false,
9684            )
9685            .await
9686            {
9687                warn!(module_id=%spec.module_id, %error, "approved configuration restart could not be scheduled");
9688            }
9689            true
9690        }
9691        SupervisorCommand::ConfigExitHold { reply } => {
9692            // A notification owns no start, backoff, reload reply or previous-
9693            // instance wait. Only an idle, registered initial start has no work
9694            // to preserve; its initialized startup context can be parked here.
9695            let idle_start = child.is_none()
9696                && lock_snapshot(snapshot).is_ok_and(|state| {
9697                    state.enabled
9698                        && state.state == ModuleState::Starting
9699                        && !state.process_alive
9700                        && !state.respawn_pending
9701                        && state.waiting_for_previous_pid.is_none()
9702                        && state.waiting_for_previous_detail.is_none()
9703                        && state.failed_retry.pending.is_none()
9704                })
9705                && runtime
9706                    .scheduled_respawn
9707                    .lock()
9708                    .unwrap_or_else(|p| p.into_inner())
9709                    .is_none()
9710                && runtime
9711                    .deferred_reload_reply
9712                    .lock()
9713                    .unwrap_or_else(|p| p.into_inner())
9714                    .is_none();
9715            let result = if idle_start {
9716                try_park_config_attempt(
9717                    spec,
9718                    runtime,
9719                    snapshot,
9720                    &runtime.failure_context(runtime.recovery_is_daemon),
9721                    None,
9722                )
9723                .map(|_| ())
9724            } else {
9725                Ok(())
9726            };
9727            let _ = reply.send(result);
9728            true
9729        }
9730        SupervisorCommand::ConfigDeliveryFailed { connection, hold } => {
9731            if registry
9732                .get_module(&spec.module_id)
9733                .ok()
9734                .flatten()
9735                .is_none_or(|registration| registration.connection_id != connection)
9736            {
9737                return true;
9738            }
9739            runtime.recovery_is_daemon = true;
9740            // With a recoverable generation, delivery-failure recovery spends
9741            // one crash-budget entry: the module rejected its configuration.
9742            // Approved configuration restarts spend none.
9743            if hold {
9744                let notice = begin_forwarding_drain_if_configured(
9745                    spec,
9746                    runtime,
9747                    registry,
9748                    snapshot,
9749                    Some(true),
9750                    RouteCloseReason::Disable,
9751                )
9752                .await;
9753                if let Ok(notice) = notice {
9754                    let drained = drain_optional_child(
9755                        &spec.module_id,
9756                        spec.protocol,
9757                        notice,
9758                        registry,
9759                        runtime.forwarding.as_deref(),
9760                        snapshot,
9761                        &runtime.terminal_ring,
9762                        &runtime.spawn_events,
9763                        child,
9764                        runtime.drain_timeout,
9765                        ModuleState::Failed,
9766                        Some(true),
9767                        &runtime.failure_context(true),
9768                    )
9769                    .await;
9770                    if drained.is_ok() {
9771                        match try_park_config_attempt(
9772                            spec,
9773                            runtime,
9774                            snapshot,
9775                            &runtime.failure_context(true),
9776                            None,
9777                        ) {
9778                            Ok(true) => {}
9779                            Ok(false) => {
9780                                if let Err(error) = budgeted_restart_child(
9781                                    spec,
9782                                    runtime,
9783                                    registry,
9784                                    process_liveness,
9785                                    snapshot,
9786                                    child,
9787                                    SupervisorHealthStatus::Unknown,
9788                                    Some("configuration delivery failed after its hold ended"),
9789                                    unix_ms_now(),
9790                                    false,
9791                                )
9792                                .await
9793                                {
9794                                    warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9795                                }
9796                            }
9797                            Err(error) => {
9798                                warn!(module_id=%spec.module_id, %error, "configuration failure could not be parked")
9799                            }
9800                        }
9801                    }
9802                }
9803            } else if let Err(error) = budgeted_restart_child(
9804                spec,
9805                runtime,
9806                registry,
9807                process_liveness,
9808                snapshot,
9809                child,
9810                SupervisorHealthStatus::Unknown,
9811                Some("configuration delivery failed"),
9812                unix_ms_now(),
9813                false,
9814            )
9815            .await
9816            {
9817                warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9818            }
9819            true
9820        }
9821        SupervisorCommand::Drain { requester, reply } => {
9822            // A plain stop runs no forwarding drain, so nothing reaches the
9823            // module over its connection before the wait: ask by signal.
9824            let result = drain_optional_child(
9825                &spec.module_id,
9826                spec.protocol,
9827                StopNotice::NotSent,
9828                registry,
9829                runtime.forwarding.as_deref(),
9830                snapshot,
9831                &runtime.terminal_ring,
9832                &runtime.spawn_events,
9833                child,
9834                runtime.drain_timeout,
9835                ModuleState::Stopped,
9836                None,
9837                &FailureContext::operator(),
9838            )
9839            .await;
9840            containment::terminate(&runtime.child_roster.containment).await;
9841            let registration_released = result.is_ok();
9842            if registration_released {
9843                runtime.record_activity_transition(
9844                    "subc.module_stopped",
9845                    &spec.module_id,
9846                    requester,
9847                );
9848            }
9849            let _ = reply.send(result);
9850            if registration_released {
9851                process_liveness.untrack_if_current(&spec.module_id, snapshot);
9852            }
9853            false
9854        }
9855        SupervisorCommand::Retire { requester, reply } => {
9856            let result = async {
9857                let stop_notice = begin_forwarding_drain_if_configured(
9858                    spec,
9859                    runtime,
9860                    registry,
9861                    snapshot,
9862                    None,
9863                    RouteCloseReason::Disable,
9864                )
9865                .await?;
9866                drain_optional_child(
9867                    &spec.module_id,
9868                    spec.protocol,
9869                    stop_notice,
9870                    registry,
9871                    runtime.forwarding.as_deref(),
9872                    snapshot,
9873                    &runtime.terminal_ring,
9874                    &runtime.spawn_events,
9875                    child,
9876                    runtime.drain_timeout,
9877                    ModuleState::Stopped,
9878                    None,
9879                    &FailureContext::operator(),
9880                )
9881                .await
9882            }
9883            .await;
9884            containment::terminate(&runtime.child_roster.containment).await;
9885            let registration_released = result.is_ok();
9886            if registration_released {
9887                runtime.record_activity_transition(
9888                    "subc.module_stopped",
9889                    &spec.module_id,
9890                    requester,
9891                );
9892            }
9893            let _ = reply.send(result);
9894            if registration_released {
9895                process_liveness.untrack_if_current(&spec.module_id, snapshot);
9896            }
9897            false
9898        }
9899        SupervisorCommand::Restart {
9900            requester,
9901            drain_timeout_ms,
9902            received_at_generation,
9903            queued_at,
9904            reply,
9905        } => {
9906            // Without this line a restart that waited in the queue (behind a
9907            // health probe cycle or another command) was invisible: the log
9908            // showed only the drain timing out, minutes after the operator's call.
9909            info!(
9910                module_id = %spec.module_id,
9911                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
9912                "restart command dequeued"
9913            );
9914            // ACK AT INITIATION, not completion. The blocking form deadlocked any
9915            // caller whose own request lane rides the module being restarted: the
9916            // caller's in-flight request keeps the drain from quiescing, the drain
9917            // keeps the restart from completing, and the completion keeps the reply
9918            // from releasing the caller — so the drain always timed out and cut the
9919            // initiator with a GOODBYE, even on a healthy module. Replying once the
9920            // restart is validated lets a self-lane caller settle, which is exactly
9921            // what makes the drain succeed. Completion is observable via
9922            // supervisor.list / module status; a post-ack failure lands the module
9923            // in a visible terminal state below rather than in a reply nobody can
9924            // receive.
9925            let validation = match lock_snapshot(snapshot) {
9926                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
9927                    module_id: spec.module_id.clone(),
9928                }),
9929                Ok(_) => Ok(()),
9930                Err(err) => Err(err),
9931            };
9932            let initiated = validation.is_ok();
9933            let _ = reply.send(validation);
9934            // A restart asks for a fresh process. Commands run one at a time,
9935            // so a restart queued behind another restart (two operator calls
9936            // in quick succession) is dequeued the moment the first one has
9937            // spawned its replacement -- before that process has sent HELLO.
9938            // Running it would drain and kill the process the first restart
9939            // just produced, which is the opposite of what both callers asked
9940            // for. If a process spawned after this request was received is
9941            // still supervised, the request is already satisfied. Not when the
9942            // configuration changed since that spawn: then the newer process
9943            // predates the spec this restart may exist to apply.
9944            // Generation publication can lag a child that is already serving
9945            // while its identity is recorded. Only process creation after request
9946            // receipt satisfies a restart, not publication after receipt.
9947            let satisfied_by_generation = if initiated
9948                && child
9949                    .as_ref()
9950                    .is_some_and(|child| child.spawned_at > queued_at)
9951            {
9952                lock_snapshot(snapshot).ok().and_then(|state| {
9953                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
9954                })
9955            } else {
9956                None
9957            };
9958            let satisfied_by_pending = initiated
9959                && child.is_none()
9960                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
9961                    // No child exists yet, so this replacement will necessarily
9962                    // be created after the queued request, unlike a published
9963                    // generation whose process may already have been serving.
9964                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
9965                    if pending {
9966                        state.coalesced_restart_pending = true;
9967                    }
9968                    pending
9969                });
9970            if satisfied_by_pending {
9971                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
9972            } else if let Some(generation) = satisfied_by_generation {
9973                info!(
9974                    module_id = %spec.module_id,
9975                    received_at_generation,
9976                    "restart already satisfied by generation {generation}; not restarting again"
9977                );
9978            } else if initiated {
9979                // Precedence: this restart's operator override, else the module's
9980                // configured budget (already resolved into the runtime).
9981                let drain_timeout = drain_timeout_ms
9982                    .map(Duration::from_millis)
9983                    .unwrap_or(runtime.drain_timeout);
9984                if let Err(err) = restart_child(
9985                    spec,
9986                    runtime,
9987                    registry,
9988                    process_liveness,
9989                    snapshot,
9990                    child,
9991                    drain_timeout,
9992                )
9993                .await
9994                {
9995                    warn!(
9996                        module_id = %spec.module_id,
9997                        error = %err,
9998                        "operator restart failed after initiation ack; module state carries the outcome"
9999                    );
10000                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10001                        state.state = ModuleState::Failed;
10002                        clear_current_process_facts(state);
10003                    });
10004                } else {
10005                    runtime.start_requester = requester;
10006                    runtime.pending_activity_restart = true;
10007                }
10008            }
10009            true
10010        }
10011        SupervisorCommand::Reload { requester, reply } => {
10012            let result =
10013                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
10014            if result.is_ok() {
10015                runtime.start_requester = requester;
10016                runtime.pending_activity_restart = true;
10017            }
10018            if result.is_ok()
10019                && runtime
10020                    .scheduled_respawn
10021                    .lock()
10022                    .unwrap_or_else(|p| p.into_inner())
10023                    .as_ref()
10024                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
10025            {
10026                *runtime
10027                    .deferred_reload_reply
10028                    .lock()
10029                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
10030            } else {
10031                let _ = reply.send(result);
10032            }
10033            true
10034        }
10035        SupervisorCommand::SetEnabled {
10036            enabled,
10037            requester,
10038            reply,
10039            reset_budget,
10040        } => {
10041            let previous_requester = runtime.start_requester.clone();
10042            if enabled {
10043                runtime.start_requester = requester.clone();
10044            }
10045            let result = set_child_enabled_with_budget_policy(
10046                spec,
10047                runtime,
10048                registry,
10049                process_liveness,
10050                snapshot,
10051                child,
10052                enabled,
10053                &runtime.failure_context(false),
10054                reset_budget,
10055            )
10056            .await;
10057            if !enabled {
10058                containment::terminate(&runtime.child_roster.containment).await;
10059            }
10060            if matches!(result, Ok(true)) {
10061                runtime.pending_activity_restart = false;
10062                if enabled {
10063                    runtime.start_requester = requester;
10064                } else {
10065                    runtime.start_requester = None;
10066                    runtime.record_activity_transition(
10067                        "subc.module_disabled",
10068                        &spec.module_id,
10069                        requester,
10070                    );
10071                }
10072            } else {
10073                runtime.start_requester = previous_requester;
10074            }
10075            let _ = reply.send(result);
10076            true
10077        }
10078        SupervisorCommand::UpdateConfiguration {
10079            spec: next_spec,
10080            health,
10081            drain_timeout_ms,
10082            reply,
10083        } => {
10084            if runtime
10085                .config_delivery
10086                .as_ref()
10087                .is_some_and(|delivery| delivery.0.file_configuration_is_gated(&spec.module_id))
10088            {
10089                let _ = reply.send(Err(SuperviseError::ConfigHeld {
10090                    module_id: spec.module_id.clone(),
10091                    code: "config_gated".into(),
10092                    detail: "live file reconfiguration cannot replace stamped authority".into(),
10093                }));
10094                return true;
10095            }
10096            stage_supervisor_configuration(
10097                spec,
10098                runtime,
10099                snapshot,
10100                next_spec,
10101                health,
10102                drain_timeout_ms,
10103            );
10104            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10105                state.configuration_updated_since_spawn = true;
10106            });
10107            let _ = reply.send(Ok(()));
10108            true
10109        }
10110        SupervisorCommand::Swap {
10111            requester,
10112            ready_timeout,
10113            reply,
10114        } => {
10115            if spec.carry_containment
10116                || containment::current(&runtime.child_roster.containment).is_some()
10117            {
10118                let _ = reply.send(Err(SuperviseError::InvalidSpec {
10119                    reason: "a carried container cannot overlap a swap candidate".to_owned(),
10120                }));
10121                return true;
10122            }
10123            let mut swap_runtime = runtime.clone();
10124            swap_runtime.start_requester = requester;
10125            let before = lock_snapshot(snapshot)
10126                .ok()
10127                .map(|state| state.spawn_generation);
10128            let end = swap::run_swap(
10129                spec,
10130                &swap_runtime,
10131                registry,
10132                process_liveness,
10133                snapshot,
10134                child,
10135                commands,
10136                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
10137                reply,
10138            )
10139            .await;
10140            if before
10141                != lock_snapshot(snapshot)
10142                    .ok()
10143                    .map(|state| state.spawn_generation)
10144            {
10145                runtime.start_requester = swap_runtime.start_requester;
10146            }
10147            requeued.extend(end.requeue);
10148            true
10149        }
10150    }
10151}
10152
10153fn stage_supervisor_configuration(
10154    spec: &mut ModuleSpec,
10155    runtime: &mut SupervisorRuntimeConfig,
10156    snapshot: &SharedSnapshot,
10157    next_spec: ModuleSpec,
10158    health: HealthConfig,
10159    drain_timeout_ms: Option<u64>,
10160) {
10161    if let Some(handle) = &runtime.supervisor_handle {
10162        handle.apply_identity_configuration(&next_spec);
10163    }
10164    let launch_changed = *spec != next_spec;
10165    *spec = next_spec;
10166    if launch_changed {
10167        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10168            state.configuration_updated_since_spawn = true
10169        });
10170    }
10171    let health_changed = runtime.health != health;
10172    runtime.health = health;
10173    // Reset the cadence and old endpoint's failure streak on a live
10174    // health-policy change rather than waiting for its old deadline.
10175    if health_changed {
10176        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10177            state.health = ModuleHealthStatus::default();
10178            state.last_probe_evidence = None;
10179        });
10180    }
10181    runtime.drain_timeout = drain_timeout_ms
10182        .map(Duration::from_millis)
10183        .unwrap_or(runtime.default_drain_timeout);
10184    *runtime
10185        .effective_drain_timeout
10186        .lock()
10187        .unwrap_or_else(|p| p.into_inner()) = runtime.drain_timeout;
10188}
10189
10190async fn restart_child(
10191    spec: &ModuleSpec,
10192    runtime: &SupervisorRuntimeConfig,
10193    registry: &Registry,
10194    process_liveness: &SupervisorProcessLiveness,
10195    snapshot: &SharedSnapshot,
10196    child: &mut Option<SupervisedChild>,
10197    drain_timeout: Duration,
10198) -> Result<(), SuperviseError> {
10199    restart_child_with_budget_policy(
10200        spec,
10201        runtime,
10202        registry,
10203        process_liveness,
10204        snapshot,
10205        child,
10206        drain_timeout,
10207        true,
10208    )
10209    .await
10210}
10211
10212#[allow(clippy::too_many_arguments)]
10213async fn restart_child_with_budget_policy(
10214    spec: &ModuleSpec,
10215    runtime: &SupervisorRuntimeConfig,
10216    registry: &Registry,
10217    process_liveness: &SupervisorProcessLiveness,
10218    snapshot: &SharedSnapshot,
10219    child: &mut Option<SupervisedChild>,
10220    drain_timeout: Duration,
10221    reset_budget: bool,
10222) -> Result<(), SuperviseError> {
10223    // Restart cycles a running module; it must not silently start a disabled one.
10224    if !lock_snapshot(snapshot)?.enabled {
10225        return Err(SuperviseError::Disabled {
10226            module_id: spec.module_id.clone(),
10227        });
10228    }
10229    let stop_notice = begin_forwarding_drain_with_timeout(
10230        spec,
10231        runtime,
10232        registry,
10233        snapshot,
10234        None,
10235        RouteCloseReason::Restart,
10236        drain_timeout,
10237    )
10238    .await?;
10239
10240    if child.is_some() {
10241        drain_optional_child(
10242            &spec.module_id,
10243            spec.protocol,
10244            stop_notice,
10245            registry,
10246            runtime.forwarding.as_deref(),
10247            snapshot,
10248            &runtime.terminal_ring,
10249            &runtime.spawn_events,
10250            child,
10251            drain_timeout,
10252            ModuleState::Restarting,
10253            Some(true),
10254            &FailureContext::operator(),
10255        )
10256        .await?;
10257    } else {
10258        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10259            state.enabled = true;
10260            state.state = ModuleState::Restarting;
10261            clear_current_process_facts(state);
10262        })?;
10263        release_dead_registration(
10264            registry,
10265            runtime.forwarding.as_deref(),
10266            snapshot,
10267            &spec.module_id,
10268            &runtime.failure_context(false),
10269        )
10270        .await?;
10271    }
10272
10273    if reset_budget {
10274        reset_restart_count(snapshot, &spec.module_id)?;
10275    }
10276    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10277    schedule_respawn(
10278        runtime,
10279        snapshot,
10280        &spec.module_id,
10281        runtime.restart_policy.backoff,
10282        RespawnKind::Spawn,
10283        false,
10284    )
10285}
10286
10287async fn reload_child(
10288    spec: &ModuleSpec,
10289    runtime: &SupervisorRuntimeConfig,
10290    registry: &Registry,
10291    process_liveness: &SupervisorProcessLiveness,
10292    snapshot: &SharedSnapshot,
10293    child: &mut Option<SupervisedChild>,
10294) -> Result<(), SuperviseError> {
10295    // Reload cycles a running module; it must not silently start a disabled one.
10296    if !lock_snapshot(snapshot)?.enabled {
10297        return Err(SuperviseError::Disabled {
10298            module_id: spec.module_id.clone(),
10299        });
10300    }
10301    let stop_notice = begin_forwarding_drain(
10302        spec,
10303        runtime,
10304        registry,
10305        snapshot,
10306        Some(true),
10307        RouteCloseReason::Reload,
10308    )
10309    .await?;
10310
10311    if child.is_some() {
10312        drain_optional_child(
10313            &spec.module_id,
10314            spec.protocol,
10315            stop_notice,
10316            registry,
10317            runtime.forwarding.as_deref(),
10318            snapshot,
10319            &runtime.terminal_ring,
10320            &runtime.spawn_events,
10321            child,
10322            runtime.drain_timeout,
10323            ModuleState::Restarting,
10324            Some(true),
10325            &FailureContext::operator(),
10326        )
10327        .await?;
10328    } else {
10329        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10330            state.enabled = true;
10331            state.state = ModuleState::Restarting;
10332            clear_current_process_facts(state);
10333        })?;
10334        release_dead_registration(
10335            registry,
10336            runtime.forwarding.as_deref(),
10337            snapshot,
10338            &spec.module_id,
10339            &runtime.failure_context(false),
10340        )
10341        .await?;
10342    }
10343
10344    reset_restart_count(snapshot, &spec.module_id)?;
10345    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10346    schedule_respawn(
10347        runtime,
10348        snapshot,
10349        &spec.module_id,
10350        runtime.restart_policy.backoff,
10351        RespawnKind::Reload,
10352        false,
10353    )
10354}
10355
10356async fn finish_reload_child(
10357    spec: &ModuleSpec,
10358    runtime: &SupervisorRuntimeConfig,
10359    registry: &Registry,
10360    process_liveness: &SupervisorProcessLiveness,
10361    snapshot: &SharedSnapshot,
10362    child: &mut Option<SupervisedChild>,
10363) -> Result<(), SuperviseError> {
10364    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10365    let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10366        Ok(next_child) => next_child,
10367        Err(error @ SuperviseError::ConfigDeliveryDeferred { .. }) => {
10368            defer_config_spawn(
10369                spec,
10370                runtime,
10371                snapshot,
10372                RespawnKind::Reload,
10373                &runtime.failure_context(runtime.recovery_is_daemon),
10374            )?;
10375            return Err(error);
10376        }
10377        Err(error @ SuperviseError::ConfigHeld { .. }) => {
10378            let failure = runtime.failure_context(runtime.recovery_is_daemon);
10379            if try_park_config_attempt(spec, runtime, snapshot, &failure, None)? {
10380                return Err(error);
10381            }
10382            defer_config_spawn(spec, runtime, snapshot, RespawnKind::Reload, &failure)?;
10383            return Err(SuperviseError::ConfigDeliveryDeferred {
10384                module_id: spec.module_id.clone(),
10385            });
10386        }
10387        Err(err) => {
10388            return handle_reload_spawn_failure(
10389                spec,
10390                runtime,
10391                process_liveness,
10392                snapshot,
10393                child,
10394                format!("new child failed to spawn: {err}"),
10395            )
10396            .await;
10397        }
10398    };
10399    *child = Some(next_child);
10400
10401    let wait_outcome = {
10402        let active_child = child.as_mut().expect("new reload child was just stored");
10403        wait_for_registration_after_reload(
10404            registry,
10405            &spec.module_id,
10406            snapshot,
10407            active_child,
10408            REGISTRY_RELEASE_TIMEOUT,
10409        )
10410        .await?
10411    };
10412
10413    match wait_outcome {
10414        RegistrationWaitOutcome::Registered => {
10415            child
10416                .as_mut()
10417                .expect("registered reload child")
10418                .registration_deadline = None;
10419            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
10420            Ok(())
10421        }
10422        RegistrationWaitOutcome::Exited(exit_report) => {
10423            if let Some(active_child) = child.as_mut() {
10424                active_child.drain_stderr(&spec.module_id).await;
10425            }
10426            // Keep the exited child's roster guard until its terminal is written.
10427            // Shutdown waits on that guard, not on the child Option used for respawn.
10428            let mut exited_child = child.take().expect("exited reload child is still stored");
10429            #[cfg(test)]
10430            if let Some(gate) = &runtime.test_reload_exit_record_gate {
10431                gate.reached.notify_one();
10432                gate.resume.notified().await;
10433            }
10434            let result = handle_reload_child_registration_failure(
10435                spec,
10436                runtime,
10437                registry,
10438                process_liveness,
10439                snapshot,
10440                child,
10441                ReloadRegistrationFailure {
10442                    exit_report: registration_failure_exit_report(exit_report),
10443                    reason: exited_child
10444                        .spawn_failure
10445                        .clone()
10446                        .unwrap_or_else(|| "new child exited before registering".to_string()),
10447                },
10448            )
10449            .await;
10450            exited_child
10451                .finish_exit(false)
10452                .await
10453                .map_err(|source| SuperviseError::Wait {
10454                    module_id: spec.module_id.clone(),
10455                    source,
10456                })?;
10457            result
10458        }
10459        RegistrationWaitOutcome::TimedOut => {
10460            let mut timed_out_child = child
10461                .take()
10462                .expect("timed-out reload child is still running");
10463            timed_out_child
10464                .start_kill()
10465                .map_err(|source| SuperviseError::Kill {
10466                    module_id: spec.module_id.clone(),
10467                    source,
10468                })?;
10469            let status = timed_out_child
10470                .wait()
10471                .await
10472                .map_err(|source| SuperviseError::Wait {
10473                    module_id: spec.module_id.clone(),
10474                    source,
10475                })?;
10476            timed_out_child.drain_stderr(&spec.module_id).await;
10477            let result = handle_reload_child_registration_failure(
10478                spec,
10479                runtime,
10480                registry,
10481                process_liveness,
10482                snapshot,
10483                child,
10484                ReloadRegistrationFailure {
10485                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
10486                        snapshot,
10487                        &timed_out_child,
10488                        &status,
10489                    )),
10490                    reason: format!(
10491                        "new child did not register within {:?}",
10492                        REGISTRY_RELEASE_TIMEOUT
10493                    ),
10494                },
10495            )
10496            .await;
10497            timed_out_child
10498                .finish_exit(false)
10499                .await
10500                .map_err(|source| SuperviseError::Wait {
10501                    module_id: spec.module_id.clone(),
10502                    source,
10503                })?;
10504            result
10505        }
10506    }
10507}
10508
10509#[allow(clippy::too_many_arguments)]
10510async fn set_child_enabled(
10511    spec: &ModuleSpec,
10512    runtime: &SupervisorRuntimeConfig,
10513    registry: &Registry,
10514    process_liveness: &SupervisorProcessLiveness,
10515    snapshot: &SharedSnapshot,
10516    child: &mut Option<SupervisedChild>,
10517    enabled: bool,
10518    failure: &FailureContext,
10519) -> Result<bool, SuperviseError> {
10520    set_child_enabled_with_budget_policy(
10521        spec,
10522        runtime,
10523        registry,
10524        process_liveness,
10525        snapshot,
10526        child,
10527        enabled,
10528        failure,
10529        true,
10530    )
10531    .await
10532}
10533
10534#[allow(clippy::too_many_arguments)]
10535async fn set_child_enabled_with_budget_policy(
10536    spec: &ModuleSpec,
10537    runtime: &SupervisorRuntimeConfig,
10538    registry: &Registry,
10539    process_liveness: &SupervisorProcessLiveness,
10540    snapshot: &SharedSnapshot,
10541    child: &mut Option<SupervisedChild>,
10542    enabled: bool,
10543    failure: &FailureContext,
10544    reset_budget: bool,
10545) -> Result<bool, SuperviseError> {
10546    let (current_enabled, current_state, respawn_pending) = {
10547        let state = lock_snapshot(snapshot)?;
10548        (state.enabled, state.state, state.respawn_pending)
10549    };
10550    // `start` (enable on an already-enabled module) heals TERMINAL states instead
10551    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
10552    // clean (Stopped) has no live process and no other in-band recovery — the
10553    // operator's start is the explicit recovery act and resets the budget. Without
10554    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
10555    // which the 2026-07-14 aft outage proved is a trap when the failed module is
10556    // the one providing every agent's shell.
10557    let revive_terminal = enabled
10558        && current_enabled
10559        && child.is_none()
10560        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
10561            || (current_state == ModuleState::Restarting && !respawn_pending));
10562    if current_enabled == enabled && !revive_terminal {
10563        return Ok(false);
10564    }
10565
10566    if enabled {
10567        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10568            state.enabled = true;
10569            state.state = ModuleState::Starting;
10570            clear_current_process_facts(state);
10571        })?;
10572        #[cfg(test)]
10573        if runtime.test_seed_stale_facts_before_enable_spawn {
10574            update_snapshot(snapshot, Some(&spec.module_id), |state| {
10575                state.process_alive = true;
10576                state.pid = Some(41);
10577                state.spawned_at_ms = Some(42);
10578                state.spawned_from = Some(PathBuf::from("/spawned/module"));
10579                state.spawned_file_identity = Some(SpawnedFileIdentity {
10580                    device: 43,
10581                    inode: 44,
10582                    #[cfg(windows)]
10583                    windows_image: None,
10584                });
10585            })?;
10586        }
10587        release_dead_registration(
10588            registry,
10589            runtime.forwarding.as_deref(),
10590            snapshot,
10591            &spec.module_id,
10592            failure,
10593        )
10594        .await?;
10595        if reset_budget {
10596            reset_restart_count(snapshot, &spec.module_id)?;
10597        }
10598        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10599        let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10600            Ok(next_child) => next_child,
10601            Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
10602                defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
10603                return Ok(true);
10604            }
10605            Err(error @ SuperviseError::ConfigHeld { .. }) => {
10606                if let Err(parking_error) = hold_config_spawn(spec, runtime, snapshot, failure) {
10607                    process_liveness.untrack_if_current(&spec.module_id, snapshot);
10608                    return Err(parking_error);
10609                }
10610                return Err(error);
10611            }
10612            Err(err) => {
10613                fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
10614                process_liveness.untrack_if_current(&spec.module_id, snapshot);
10615                return Err(err);
10616            }
10617        };
10618        *child = Some(next_child);
10619        debug!(module_id = %spec.module_id, "supervised module enabled");
10620        Ok(true)
10621    } else {
10622        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10623            state.config_hold_parked = false;
10624            state.config_hold_daemon_initiated = false;
10625            state.config_hold_approval = None;
10626        })?;
10627        let stop_notice = begin_forwarding_drain_if_configured(
10628            spec,
10629            runtime,
10630            registry,
10631            snapshot,
10632            Some(false),
10633            RouteCloseReason::Disable,
10634        )
10635        .await?;
10636        drain_optional_child(
10637            &spec.module_id,
10638            spec.protocol,
10639            stop_notice,
10640            registry,
10641            runtime.forwarding.as_deref(),
10642            snapshot,
10643            &runtime.terminal_ring,
10644            &runtime.spawn_events,
10645            child,
10646            runtime.drain_timeout,
10647            ModuleState::Disabled,
10648            Some(false),
10649            failure,
10650        )
10651        .await?;
10652        debug!(module_id = %spec.module_id, "supervised module disabled");
10653        Ok(true)
10654    }
10655}
10656
10657#[allow(clippy::too_many_arguments)]
10658async fn on_child_exit(
10659    spec: &ModuleSpec,
10660    policy: RestartPolicy,
10661    registry: &Registry,
10662    snapshot: &SharedSnapshot,
10663    terminal_ring: &Arc<Mutex<TerminalRing>>,
10664    spawn_events: &SpawnEventFeed,
10665    roster: &ChildRoster,
10666    exit_report: ExitReport,
10667    daemon_initiated: bool,
10668) -> NextAction {
10669    // Once the daemon has begun shutting down, no exit is a crash to recover
10670    // from: the module is exiting because the daemon is going away (EOF on its
10671    // connection, or a service manager signalling the whole cgroup). Record it
10672    // as such and never schedule a respawn, which would only start a process
10673    // for the shutdown to end again.
10674    if roster.is_closed() {
10675        return on_child_exit_during_daemon_shutdown(
10676            spec,
10677            registry,
10678            snapshot,
10679            terminal_ring,
10680            spawn_events,
10681            exit_report,
10682        )
10683        .await;
10684    }
10685    // Deliberate stops the supervisor asks for (operator stop, disable,
10686    // restart, reload, swap, a health restart, a drain that runs out of budget)
10687    // takes the child out of the supervise loop and reaps it in
10688    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
10689    // that reaches this point is either unrequested or a first-HELLO timeout,
10690    // which is deliberately classified as a crash even after a clean TERM exit.
10691    //
10692    // For a subc-wire module a clean exit is still a stop: those modules are
10693    // written to re-raise SIGTERM, so a stray outside signal already reads as a
10694    // crash, and exiting 0 is a deliberate choice the module made. A
10695    // `protocol: "none"` module is a stock program we cannot change, and many
10696    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
10697    // stop would leave the module down for good after any stray signal, so it
10698    // goes through the crash path instead: it spends restart budget, respawns
10699    // with the crash backoff, and ends `failed` when the budget runs out.
10700    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
10701        && running_protocol(spec, snapshot) == ModuleProtocol::None;
10702    match exit_report.kind {
10703        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
10704            info!(
10705                module_id = %spec.module_id,
10706                exit_code = ?exit_report.code,
10707                exit_signal = ?exit_report.signal,
10708                "supervised module exited cleanly"
10709            );
10710            if let Err(err) = publish_after_terminal_record(
10711                &spec.module_id,
10712                || {
10713                    record_terminal(
10714                        &spec.module_id,
10715                        terminal_ring,
10716                        spawn_events,
10717                        &exit_report,
10718                        TerminalDisposition::Stopped,
10719                    )
10720                },
10721                || {
10722                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10723                        state.state = ModuleState::Stopped;
10724                        clear_current_process_facts(state);
10725                        state.last_exit = Some(exit_report.clone());
10726                    })
10727                },
10728            ) {
10729                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
10730            }
10731            let registration_released = match wait_for_registration_release(
10732                registry,
10733                &spec.module_id,
10734                REGISTRY_RELEASE_TIMEOUT,
10735            )
10736            .await
10737            {
10738                Ok(()) => true,
10739                Err(err) => {
10740                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
10741                    false
10742                }
10743            };
10744            NextAction::Stop {
10745                registration_released,
10746            }
10747        }
10748        ExitKind::Clean | ExitKind::Crash => {
10749            if unrequested_clean_exit_of_protocol_none {
10750                warn!(
10751                    module_id = %spec.module_id,
10752                    exit_code = ?exit_report.code,
10753                    exit_signal = ?exit_report.signal,
10754                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
10755                );
10756            } else {
10757                warn!(
10758                    module_id = %spec.module_id,
10759                    exit_code = ?exit_report.code,
10760                    exit_signal = ?exit_report.signal,
10761                    "supervised module exited abnormally (crash)"
10762                );
10763            }
10764            let mut restart_schedule = None;
10765            let mut disposition = TerminalDisposition::Disabled;
10766            let mut final_state = ModuleState::Disabled;
10767            // Set only when the budget is what stopped the module, so the
10768            // terminal record says which limit was hit rather than leaving
10769            // `failed` to be read as "crashed once, badly".
10770            let mut disposition_detail = lock_snapshot(snapshot)
10771                .ok()
10772                .and_then(|mut state| state.spawn_failure.take());
10773            let now = Instant::now();
10774            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10775                if state.enabled {
10776                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
10777                        // Reserving the last slot exhausts the budget for the
10778                        // replacement, not for this exit. Publish its scheduled
10779                        // recovery atomically with the reservation so connection
10780                        // cleanup sees it while terminal recording still holds
10781                        // the visible state at Running.
10782                        state.respawn_pending = true;
10783                        final_state = ModuleState::Restarting;
10784                        restart_schedule = Some(schedule);
10785                        disposition = TerminalDisposition::Restarting;
10786                    } else {
10787                        final_state = ModuleState::Failed;
10788                        disposition = TerminalDisposition::Failed;
10789                        let budget = policy.budget_exhausted_detail();
10790                        disposition_detail =
10791                            Some(disposition_detail.take().map_or_else(
10792                                || budget.clone(),
10793                                |cause| format!("{cause}; {budget}"),
10794                            ));
10795                    }
10796                } else {
10797                    disposition = TerminalDisposition::Disabled;
10798                }
10799            }) {
10800                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
10801                return NextAction::Stop {
10802                    registration_released: false,
10803                };
10804            }
10805            if disposition == TerminalDisposition::Failed {
10806                // The window is in the message, not only in the fields: this line
10807                // is read in a scrollback where a bare `max_restarts=3` reads as a
10808                // lifetime cap and sends the operator looking for three crashes
10809                // that never happened together.
10810                error!(
10811                    module_id = %spec.module_id,
10812                    max_restarts = policy.max_restarts,
10813                    window_secs = policy.window.as_secs(),
10814                    "module stopped: {}",
10815                    policy.budget_exhausted_detail()
10816                );
10817            }
10818            if let Err(err) = publish_after_terminal_record(
10819                &spec.module_id,
10820                || {
10821                    record_terminal_with_detail(
10822                        &spec.module_id,
10823                        terminal_ring,
10824                        spawn_events,
10825                        &exit_report,
10826                        disposition,
10827                        disposition_detail,
10828                    )
10829                },
10830                || {
10831                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10832                        state.state = final_state;
10833                        clear_current_process_facts(state);
10834                        if final_state == ModuleState::Failed
10835                            && daemon_initiated
10836                            && !roster.is_closed()
10837                        {
10838                            state.failed_retry.schedule(&spec.module_id);
10839                        }
10840                        state.last_exit = Some(exit_report.clone());
10841                    })
10842                },
10843            ) {
10844                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
10845            }
10846
10847            if let Some(schedule) = restart_schedule {
10848                NextAction::Restart {
10849                    schedule: Some(schedule),
10850                }
10851            } else {
10852                let registration_released = match wait_for_registration_release(
10853                    registry,
10854                    &spec.module_id,
10855                    REGISTRY_RELEASE_TIMEOUT,
10856                )
10857                .await
10858                {
10859                    Ok(()) => true,
10860                    Err(err) => {
10861                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
10862                        false
10863                    }
10864                };
10865                NextAction::Stop {
10866                    registration_released,
10867                }
10868            }
10869        }
10870        ExitKind::DeliberateSeverance => {
10871            warn!(
10872                module_id = %spec.module_id,
10873                exit_code = ?exit_report.code,
10874                exit_signal = ?exit_report.signal,
10875                "supervised module exited after deliberate connection severance"
10876            );
10877            let should_restart = match lock_snapshot(snapshot) {
10878                Ok(state) => state.enabled,
10879                Err(err) => {
10880                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
10881                    return NextAction::Stop {
10882                        registration_released: false,
10883                    };
10884                }
10885            };
10886            let final_state = if should_restart {
10887                ModuleState::Restarting
10888            } else {
10889                ModuleState::Disabled
10890            };
10891            if let Err(err) = publish_after_terminal_record(
10892                &spec.module_id,
10893                || {
10894                    record_terminal(
10895                        &spec.module_id,
10896                        terminal_ring,
10897                        spawn_events,
10898                        &exit_report,
10899                        terminal_disposition(final_state),
10900                    )
10901                },
10902                || {
10903                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10904                        state.state = final_state;
10905                        clear_current_process_facts(state);
10906                        state.last_exit = Some(exit_report.clone());
10907                        state.lifetime_restarts += 1;
10908                    })
10909                },
10910            ) {
10911                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
10912                return NextAction::Stop {
10913                    registration_released: false,
10914                };
10915            }
10916
10917            if should_restart {
10918                NextAction::Restart { schedule: None }
10919            } else {
10920                let registration_released = match wait_for_registration_release(
10921                    registry,
10922                    &spec.module_id,
10923                    REGISTRY_RELEASE_TIMEOUT,
10924                )
10925                .await
10926                {
10927                    Ok(()) => true,
10928                    Err(err) => {
10929                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
10930                        false
10931                    }
10932                };
10933                NextAction::Stop {
10934                    registration_released,
10935                }
10936            }
10937        }
10938    }
10939}
10940
10941async fn on_child_exit_during_daemon_shutdown(
10942    spec: &ModuleSpec,
10943    registry: &Registry,
10944    snapshot: &SharedSnapshot,
10945    terminal_ring: &Arc<Mutex<TerminalRing>>,
10946    spawn_events: &SpawnEventFeed,
10947    exit_report: ExitReport,
10948) -> NextAction {
10949    info!(
10950        module_id = %spec.module_id,
10951        exit_code = ?exit_report.code,
10952        exit_signal = ?exit_report.signal,
10953        exit_kind = ?exit_report.kind,
10954        "supervised module exited during daemon shutdown; not restarting it"
10955    );
10956    if let Err(err) = publish_after_terminal_record(
10957        &spec.module_id,
10958        || {
10959            record_terminal(
10960                &spec.module_id,
10961                terminal_ring,
10962                spawn_events,
10963                &exit_report,
10964                TerminalDisposition::DaemonShutdown,
10965            )
10966        },
10967        || {
10968            update_snapshot(snapshot, Some(&spec.module_id), |state| {
10969                state.state = ModuleState::Stopped;
10970                clear_current_process_facts(state);
10971                state.last_exit = Some(exit_report.clone());
10972            })
10973        },
10974    ) {
10975        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
10976    }
10977    let registration_released =
10978        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
10979            .await
10980            .is_ok();
10981    NextAction::Stop {
10982        registration_released,
10983    }
10984}
10985
10986// Keep the status lock out of terminal I/O. Readers may observe the previous
10987// state while recording is in progress, but never a completed exit without its
10988// history. Journal errors are logged by the journal and retained in the ring;
10989// a recording panic still publishes the final state before resuming the unwind.
10990fn publish_after_terminal_record<T>(
10991    module_id: &str,
10992    record: impl FnOnce(),
10993    publish: impl FnOnce() -> T,
10994) -> T {
10995    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
10996    let published = publish();
10997    if let Err(panic) = recorded {
10998        error!(
10999            module_id,
11000            "terminal recording panicked; published final state before resuming unwind"
11001        );
11002        std::panic::resume_unwind(panic);
11003    }
11004    published
11005}
11006
11007fn fail_after_wait_error(
11008    module_id: &str,
11009    snapshot: &SharedSnapshot,
11010    terminal_ring: &Arc<Mutex<TerminalRing>>,
11011    spawn_events: &SpawnEventFeed,
11012    detail: String,
11013    failure: &FailureContext,
11014) {
11015    let report = wait_error_exit_report();
11016    publish_after_terminal_record(
11017        module_id,
11018        || {
11019            record_terminal_with_detail(
11020                module_id,
11021                terminal_ring,
11022                spawn_events,
11023                &report,
11024                TerminalDisposition::Failed,
11025                Some(detail),
11026            )
11027        },
11028        || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
11029    );
11030}
11031
11032#[cfg(test)]
11033fn record_wait_error_terminal(
11034    module_id: &str,
11035    terminal_ring: &Arc<Mutex<TerminalRing>>,
11036    spawn_events: &SpawnEventFeed,
11037) {
11038    record_terminal(
11039        module_id,
11040        terminal_ring,
11041        spawn_events,
11042        &wait_error_exit_report(),
11043        TerminalDisposition::Failed,
11044    );
11045}
11046
11047fn record_terminal(
11048    module_id: &str,
11049    terminal_ring: &Arc<Mutex<TerminalRing>>,
11050    spawn_events: &SpawnEventFeed,
11051    exit_report: &ExitReport,
11052    disposition: TerminalDisposition,
11053) {
11054    record_terminal_with_detail(
11055        module_id,
11056        terminal_ring,
11057        spawn_events,
11058        exit_report,
11059        disposition,
11060        None,
11061    );
11062}
11063
11064/// The ring lock is held only to capture the read (see
11065/// `TerminalJournal::capture_read`), so this module's exits keep recording
11066/// while the journal files are read. Blocking: it reads files.
11067fn durable_terminal_history_of(
11068    terminal_ring: &Mutex<TerminalRing>,
11069    module_id: &str,
11070) -> subc_control::TerminalHistory {
11071    let read = terminal_ring
11072        .lock()
11073        .unwrap_or_else(|p| p.into_inner())
11074        .capture_durable_history();
11075    read.read(module_id)
11076}
11077
11078fn record_terminal_with_detail(
11079    module_id: &str,
11080    terminal_ring: &Arc<Mutex<TerminalRing>>,
11081    spawn_events: &SpawnEventFeed,
11082    exit_report: &ExitReport,
11083    disposition: TerminalDisposition,
11084    disposition_detail: Option<String>,
11085) {
11086    #[cfg(test)]
11087    terminal_history_tests::before_terminal_record(spawn_events);
11088    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
11089    let record = TerminalRecord {
11090        exit_code: exit_report.code,
11091        exit_signal: exit_report.signal,
11092        at_ms: exit_report.at_ms,
11093        disposition,
11094        exit_kind: exit_report.kind.into(),
11095        disposition_detail,
11096    };
11097    terminal_ring
11098        .lock()
11099        .unwrap_or_else(|poisoned| poisoned.into_inner())
11100        .record_exit(module_id, record);
11101}
11102
11103fn untrack_if_registration_released(
11104    process_liveness: &SupervisorProcessLiveness,
11105    registry: &Registry,
11106    module_id: &str,
11107    snapshot: &SharedSnapshot,
11108) {
11109    match registry.get_module(module_id) {
11110        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
11111        Ok(Some(_)) => {}
11112        Err(err) => {
11113            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
11114        }
11115    }
11116}
11117
11118/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
11119/// then apply the module's configured entries minus daemon-private capture keys.
11120///
11121/// Separated from `spawn_child` only so it can be asserted without spawning a
11122/// process — a duplicate of this logic in a test would pass while the real one
11123/// drifted, which is the defect class this function exists to avoid.
11124/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
11125/// nonce. A `protocol: "none"` module gets neither, because it cannot use
11126/// either and the argument would stop a stock binary from starting at all.
11127/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
11128///
11129/// The plain-spawn form, kept for the tests that assert its plan; spawns go
11130/// through [`apply_wire_spawn_args_for_admission`].
11131#[cfg(test)]
11132fn apply_wire_spawn_args(
11133    command: &mut Command,
11134    spec: &ModuleSpec,
11135    connection_file_path: Option<&std::path::Path>,
11136    handle: Option<&SupervisorHandle>,
11137) -> Result<Option<NonceHandoff>, SuperviseError> {
11138    apply_wire_spawn_args_for_role(
11139        command,
11140        spec,
11141        connection_file_path,
11142        handle,
11143        SpawnRole::Plain,
11144    )
11145}
11146
11147/// The read end of a spawn's launch-nonce pipe, prepared by
11148/// [`apply_wire_spawn_args_for_admission`] and installed as the child's descriptor 3
11149/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
11150/// server before spawn and authenticates its client after the child exists.
11151#[cfg(unix)]
11152type NonceHandoff = subc_os::LaunchNonceHandoff;
11153#[cfg(windows)]
11154type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
11155#[cfg(not(any(unix, windows)))]
11156type NonceHandoff = std::convert::Infallible;
11157
11158/// Create the launch nonce that a new module process must present when it
11159/// registers or opens a consumer connection, including a replacement process
11160/// started alongside a still-serving module.
11161///
11162/// A plain spawn replaces the module's recorded nonce. A swap candidate records
11163/// a separate candidate token so the still-serving incumbent and its consumers
11164/// keep their nonce. Both records are installed before the process exists, so
11165/// the child's initial HELLO registration cannot arrive ahead of its nonce.
11166///
11167/// On Unix the nonce is delivered only through a pipe. It is written into
11168/// a pipe whose read end the child gets as descriptor 3, named by
11169/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
11170/// process of the same user cannot read it with `ps eww`. That handoff is
11171/// returned rather than installed here. The combined descriptor placement runs
11172/// after the cgroup callback and placement barrier, so replacing slots 3-5
11173/// cannot close a descriptor either callback still needs.
11174/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
11175#[cfg(test)]
11176fn apply_wire_spawn_args_for_role(
11177    command: &mut Command,
11178    spec: &ModuleSpec,
11179    connection_file_path: Option<&std::path::Path>,
11180    handle: Option<&SupervisorHandle>,
11181    role: SpawnRole,
11182) -> Result<Option<NonceHandoff>, SuperviseError> {
11183    apply_wire_spawn_args_for_admission(command, spec, connection_file_path, handle, role, None)
11184}
11185
11186fn apply_wire_spawn_args_for_admission(
11187    command: &mut Command,
11188    spec: &ModuleSpec,
11189    connection_file_path: Option<&std::path::Path>,
11190    handle: Option<&SupervisorHandle>,
11191    role: SpawnRole,
11192    admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11193) -> Result<Option<NonceHandoff>, SuperviseError> {
11194    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
11195    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
11196    // included: a daemon started from a module's process tree inherits it,
11197    // and passing it on would point the child at a descriptor it does not
11198    // have.
11199    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
11200    #[cfg(windows)]
11201    {
11202        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
11203        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
11204    }
11205    // Remove inherited or configured copies too: withholding must mean absent.
11206    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
11207    command.env_remove(subc_os::registration_channel::REGISTRATION_FD_ENV);
11208    command.env_remove(subc_os::registration_channel::REGISTRATION_PIPE_ENV);
11209    command.env_remove(subc_os::registration_channel::DAEMON_PID_ENV);
11210    if role == SpawnRole::Plain {
11211        if let Some(handle) = handle {
11212            #[cfg(all(test, unix))]
11213            {
11214                let pause = {
11215                    let mut slot = handle.plain_identity_policy_pause.lock().unwrap();
11216                    if slot
11217                        .as_ref()
11218                        .is_some_and(|(module, _)| module == &spec.module_id)
11219                    {
11220                        slot.take().map(|(_, pause)| pause)
11221                    } else {
11222                        None
11223                    }
11224                };
11225                if let Some(pause) = pause {
11226                    pause.reached.send(()).unwrap();
11227                    pause.release.recv().unwrap();
11228                }
11229            }
11230            // A rescan preserves the running child's nonce and reservation.
11231            // Replacement spawn preparation applies the new protocol now,
11232            // before protocol:none can return without minting a nonce.
11233            handle.apply_identity_policy(spec, spec.protocol);
11234        }
11235    }
11236    if spec.protocol == ModuleProtocol::None {
11237        return Ok(None);
11238    }
11239    if let Some(connection_file_path) = connection_file_path {
11240        command.arg(SUBC_ARG).arg(connection_file_path);
11241    }
11242
11243    if admission.is_some_and(|admission| admission.launch.is_some()) {
11244        if let Some(handle) = handle {
11245            match role {
11246                SpawnRole::Plain => {
11247                    handle
11248                        .spawn_nonces
11249                        .lock()
11250                        .unwrap_or_else(|p| p.into_inner())
11251                        .remove(&spec.module_id);
11252                    if spec.reserved {
11253                        handle
11254                            .reserved_nonces
11255                            .lock()
11256                            .unwrap_or_else(|p| p.into_inner())
11257                            .insert(spec.module_id.clone(), None);
11258                    }
11259                }
11260                // Swaps still need their lifecycle record, but the empty value
11261                // is not a bearer token and is never accepted by nonce HELLO.
11262                SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, String::new()),
11263            }
11264        }
11265        return Ok(None);
11266    }
11267    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
11268    // route.open attestation. Reserved modules additionally use the same nonce
11269    // for HELLO id-squatting protection. A respawn rotates both records.
11270    let nonce = generate_launch_nonce()?;
11271    if let Some(handle) = handle {
11272        match role {
11273            SpawnRole::Plain => {
11274                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
11275                if spec.reserved {
11276                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
11277                }
11278            }
11279            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
11280        }
11281    }
11282    #[cfg(unix)]
11283    let handoff = {
11284        let handoff =
11285            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
11286                program: spec.program.clone(),
11287                source,
11288                cgroup_path: None,
11289            })?;
11290        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
11291        Some(handoff)
11292    };
11293    #[cfg(windows)]
11294    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
11295    #[cfg(not(any(unix, windows)))]
11296    let handoff = None;
11297    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
11298    // until the daemon's per-spawn delivery records show that every running
11299    // Windows module has read its secret from the named pipe.
11300    #[cfg(not(unix))]
11301    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
11302    Ok(handoff)
11303}
11304
11305pub(crate) fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
11306    command.env_remove(CK_LOG_ENV);
11307    // The spawn role is the supervisor's to set, and only on a swap candidate
11308    // (see `apply_spawn_role`). Removing it here, rather than just not setting
11309    // it, is what makes it absent on a plain spawn: the daemon's own
11310    // environment could carry it, and so could a spec built outside daemon
11311    // config (config refuses it as an `env` key). A module reading it on a
11312    // plain restart would pick the long swap budget and leave callers waiting.
11313    command.env_remove(SUBC_SPAWN_ROLE_ENV);
11314    for (key, value) in &spec.env {
11315        // cortexkit-log currently exposes retention only as a Rust struct, not
11316        // environment names. These values are daemon-private sink metadata and
11317        // must never become a public child-process contract by being inherited.
11318        if matches!(
11319            key.as_str(),
11320            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
11321        ) || key == SUBC_SPAWN_ROLE_ENV
11322        {
11323            continue;
11324        }
11325        command.env(key, value);
11326    }
11327}
11328
11329#[cfg(windows)]
11330fn prepare_windows_registration_pipe(
11331    command: &mut Command,
11332    spec: &ModuleSpec,
11333) -> Result<Option<SpawnRegistrationChannel>, SuperviseError> {
11334    if spec.protocol == ModuleProtocol::None {
11335        return Ok(None);
11336    }
11337    let pipe = subc_os::registration_channel::RegistrationPipe::new().map_err(|source| {
11338        SuperviseError::Spawn {
11339            program: spec.program.clone(),
11340            source,
11341            cgroup_path: None,
11342        }
11343    })?;
11344    command
11345        .env(
11346            subc_os::registration_channel::REGISTRATION_PIPE_ENV,
11347            pipe.name(),
11348        )
11349        .env(
11350            subc_os::registration_channel::DAEMON_PID_ENV,
11351            std::process::id().to_string(),
11352        );
11353    Ok(Some(pipe))
11354}
11355
11356#[cfg(windows)]
11357fn prepare_windows_nonce_pipe(
11358    command: &mut Command,
11359    spec: &ModuleSpec,
11360    nonce: &str,
11361) -> Option<NonceHandoff> {
11362    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
11363        Ok(handoff) => {
11364            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
11365            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
11366            // still register until every module reads the pipe. A reader without
11367            // this variable set to env returns the pipe error instead of using
11368            // an inherited or configured environment copy.
11369            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
11370            Some(handoff)
11371        }
11372        Err(error) => {
11373            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
11374            None
11375        }
11376    }
11377}
11378
11379#[cfg(windows)]
11380fn start_windows_nonce_pipe(
11381    handoff: Option<NonceHandoff>,
11382    pid: u32,
11383    spec: &ModuleSpec,
11384) -> Option<subc_os::LaunchNoncePipeDelivery> {
11385    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
11386    // same maximum wait used for registration during a reload. Older modules
11387    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
11388    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
11389        Ok(delivery) => Some(delivery),
11390        Err(error) => {
11391            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
11392            None
11393        }
11394    })
11395}
11396
11397fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
11398    #[cfg(windows)]
11399    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
11400        return Some(
11401            if snapshot
11402                .pipe_consumed
11403                .as_ref()
11404                .is_some_and(|consumed| consumed.get().is_some())
11405            {
11406                "pipe"
11407            } else {
11408                "env"
11409            }
11410            .to_owned(),
11411        );
11412    }
11413    let _ = snapshot;
11414    None
11415}
11416
11417/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
11418/// of a blue/green swap.
11419#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11420enum SpawnRole {
11421    Plain,
11422    SwapCandidate,
11423}
11424
11425/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
11426/// `apply_child_env` has already removed the variable for every spawn.
11427fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
11428    if role == SpawnRole::SwapCandidate {
11429        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
11430    }
11431}
11432
11433fn previous_instance_error(
11434    spec: &ModuleSpec,
11435    pid: u32,
11436    reason: crate::live_children::RetainedReason,
11437) -> SuperviseError {
11438    SuperviseError::Spawn {
11439        program: spec.program.clone(),
11440        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
11441        cgroup_path: None,
11442    }
11443}
11444
11445fn carries_containment(configured: bool, has_container: bool) -> bool {
11446    configured || has_container
11447}
11448
11449fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
11450    if !carries_containment(
11451        spec.carry_containment,
11452        containment::current(&roster.containment).is_some(),
11453    ) {
11454        return None;
11455    }
11456    Some(SuperviseError::Spawn {
11457        program: spec.program.clone(),
11458        source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
11459        cgroup_path: None,
11460    })
11461}
11462
11463#[allow(clippy::too_many_arguments)]
11464fn spawn_child(
11465    spec: &ModuleSpec,
11466    connection_file_path: Option<&std::path::Path>,
11467    handle: Option<&SupervisorHandle>,
11468    ring: &Arc<Mutex<StderrRing>>,
11469    capture_logs_dir: Option<&std::path::Path>,
11470    roster: &ChildRoster,
11471    config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11472    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11473    register_timeout: Duration,
11474) -> Result<SupervisedChild, SuperviseError> {
11475    #[cfg(all(test, unix))]
11476    if let Some(source) = containment_tests::join_failure(spec, roster) {
11477        containment::terminate_blocking(&roster.containment);
11478        return Err(SuperviseError::Spawn {
11479            program: spec.program.clone(),
11480            source,
11481            cgroup_path: None,
11482        });
11483    }
11484    let result = spawn_child_in_slot(
11485        spec,
11486        connection_file_path,
11487        handle,
11488        ring,
11489        capture_logs_dir,
11490        roster,
11491        config_delivery,
11492        #[cfg(target_os = "linux")]
11493        cgroup_placement,
11494        SpawnRole::Plain,
11495        false,
11496        register_timeout,
11497    );
11498    if result
11499        .as_ref()
11500        .is_err_and(|error| !is_launch_deferral(error))
11501    {
11502        if roster.admission_pending(&spec.module_id) {
11503            containment::terminate_deferred(&roster.containment);
11504        } else {
11505            containment::terminate_blocking(&roster.containment);
11506        }
11507    }
11508    result
11509}
11510
11511/// A configuration deferral means the launch never started: the module gate
11512/// or the configuration owner was briefly busy, and every caller schedules
11513/// the same launch again shortly. A carried container stays for that retry,
11514/// as it does across a restart backoff; ending it here would kill the
11515/// module's helpers and give up the group the retry is meant to join.
11516fn is_launch_deferral(error: &SuperviseError) -> bool {
11517    matches!(error, SuperviseError::ConfigDeliveryDeferred { .. })
11518}
11519
11520/// Spawn one process of `spec` into a slot.
11521///
11522/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
11523/// A swap candidate needs a different cgroup from the process it is replacing,
11524/// which is still alive: in the same cgroup the two would be one kill domain,
11525/// and killing a failed candidate could take the incumbent with it.
11526///
11527/// The stderr capture file is `<module_id>.stderr.log` for every process of
11528/// the module, whichever slot it is in, because that is the one file
11529/// `ck module logs` reads. During a swap's overlap both processes append to it;
11530/// the daemon writes whole lines, so the two interleave by line, which is also
11531/// the merged view an operator wants while a swap runs.
11532#[allow(clippy::too_many_arguments)]
11533fn spawn_child_in_slot(
11534    spec: &ModuleSpec,
11535    connection_file_path: Option<&std::path::Path>,
11536    handle: Option<&SupervisorHandle>,
11537    ring: &Arc<Mutex<StderrRing>>,
11538    capture_logs_dir: Option<&std::path::Path>,
11539    roster: &ChildRoster,
11540    config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11541    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11542    role: SpawnRole,
11543    alternate_slot: bool,
11544    register_timeout: Duration,
11545) -> Result<SupervisedChild, SuperviseError> {
11546    let prepared = prepare_child_in_slot(
11547        spec,
11548        connection_file_path,
11549        handle,
11550        capture_logs_dir,
11551        roster,
11552        config_delivery,
11553        #[cfg(target_os = "linux")]
11554        cgroup_placement,
11555        role,
11556        alternate_slot,
11557        false,
11558    )?;
11559    let gate = config_delivery
11560        .map(|delivery| delivery.module_gate(spec))
11561        .transpose()?;
11562    let _gate = gate
11563        .as_ref()
11564        .map(|gate| {
11565            gate.try_lock()
11566                .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11567                    module_id: spec.module_id.clone(),
11568                })
11569        })
11570        .transpose()?;
11571    if let (Some(delivery), Some(admission)) = (config_delivery, &prepared.spawn_admission) {
11572        delivery.check(spec, admission)?;
11573    }
11574    let checked_base = prepared
11575        .spawn_admission
11576        .as_ref()
11577        .and_then(|admission| admission.approved_base())
11578        .cloned();
11579    let mut result = launch_child_in_slot(spec, ring, roster, prepared, register_timeout);
11580    drop(_gate);
11581    if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11582        (config_delivery, &result)
11583    {
11584        if let Some(base) = checked_base.as_ref().filter(|_| code == "program_refused") {
11585            if !delivery.record_refusal(spec, base, detail)? {
11586                result = Err(spawn_plan_stale(spec));
11587            }
11588        }
11589    }
11590    result
11591}
11592
11593struct AsyncInitialIdentity<'a> {
11594    supervisor: &'a Supervisor,
11595    module: String,
11596    committed: bool,
11597}
11598impl Drop for AsyncInitialIdentity<'_> {
11599    fn drop(&mut self) {
11600        if !self.committed {
11601            self.supervisor.abandon_unrostered(&self.module);
11602        }
11603    }
11604}
11605
11606/// Child-free preparation can be abandoned without orphaning a process. After
11607/// this value returns, the async caller validates its launch witness and creates
11608/// and publishes the child without another await.
11609struct PreparedChild {
11610    script_delivery: Option<script::Delivery>,
11611    registration_spawn: Option<Arc<RegistrationSpawn>>,
11612    registration_handle: Option<SupervisorHandle>,
11613    #[cfg(any(unix, windows))]
11614    registration_channel: Option<SpawnRegistrationChannel>,
11615    config_context: Option<config_delivery::ConfigDelivery>,
11616    binding_publication: Option<ProgramBindingPublication>,
11617    launch_spec: ModuleSpec,
11618    spawn_admission: Option<crate::config_gate::runtime::SpawnAdmission>,
11619    command: Command,
11620    carrying: bool,
11621    carried: Option<Arc<containment::CarriedContainment>>,
11622    #[cfg(unix)]
11623    launch_identity: Option<containment::LaunchIdentity>,
11624    #[cfg(unix)]
11625    group_target: containment::GroupTarget,
11626    #[cfg(target_os = "linux")]
11627    cgroup_placement: Option<subc_cgroup::Placement>,
11628    #[cfg(target_os = "linux")]
11629    cgroup_name: String,
11630    #[cfg(target_os = "linux")]
11631    cgroup_cleanup: PreparedCgroup,
11632    cgroup_path: Option<PathBuf>,
11633    #[cfg(unix)]
11634    admission_gate: Option<admission::Gate>,
11635    output_sink: ChildOutputSink,
11636    #[cfg(target_os = "macos")]
11637    privacy_exec: Option<PrivacyExec>,
11638    #[cfg(windows)]
11639    image_capture: subc_os::ExecutableCapture,
11640    #[cfg(windows)]
11641    nonce_handoff: Option<NonceHandoff>,
11642    spawned_at: Instant,
11643    config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
11644    delivery_registration: Option<config_delivery::RegistrationGuard>,
11645    program_binding: Option<SpawnProgramBinding>,
11646}
11647
11648#[cfg(target_os = "linux")]
11649struct PreparedCgroup(Option<(subc_cgroup::Placement, String)>);
11650#[cfg(target_os = "linux")]
11651impl Drop for PreparedCgroup {
11652    fn drop(&mut self) {
11653        if let Some((placement, name)) = &self.0 {
11654            remove_module_cgroup(placement, name);
11655        }
11656    }
11657}
11658
11659pub(crate) struct SpawnProgramBinding {
11660    pub(crate) path: PathBuf,
11661    // Keep the inode alive if the configured name is replaced while preparation
11662    // is suspended, so replacement cannot pass through inode reuse.
11663    _file: std::fs::File,
11664    pub(crate) identity: subc_os::RetainedFileIdentity,
11665}
11666
11667impl SpawnProgramBinding {
11668    pub(crate) fn capture(spec: &ModuleSpec) -> Result<Self, SuperviseError> {
11669        Self::capture_after_resolution(spec, |_| {})
11670    }
11671
11672    fn capture_after_resolution(
11673        spec: &ModuleSpec,
11674        before_open: impl FnOnce(&std::path::Path),
11675    ) -> Result<Self, SuperviseError> {
11676        require_program_search_path(spec)?;
11677        #[cfg(windows)]
11678        let windows_capture =
11679            windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
11680                program: spec.program.clone(),
11681                source,
11682                cgroup_path: None,
11683            })?;
11684        #[cfg(windows)]
11685        let path = windows_capture.path().to_path_buf();
11686        #[cfg(not(windows))]
11687        let path = if spec.program.is_absolute() || spec.program.components().count() > 1 {
11688            spec.program.clone()
11689        } else {
11690            let path = spec
11691                .env
11692                .iter()
11693                .rev()
11694                .find(|(key, _)| key == "PATH")
11695                .map(|(_, path)| std::ffi::OsString::from(path))
11696                .or_else(|| std::env::var_os("PATH"))
11697                .unwrap_or_else(|| {
11698                    if cfg!(target_os = "macos") {
11699                        // Match Darwin's spawn search when PATH is absent, not
11700                        // the current-directory search of an explicitly empty PATH.
11701                        "/usr/bin:/bin".into()
11702                    } else {
11703                        Default::default()
11704                    }
11705                });
11706            std::env::split_paths(&path)
11707                .map(|dir| dir.join(&spec.program))
11708                .find(|path| {
11709                    if !path.is_file() {
11710                        return false;
11711                    }
11712                    #[cfg(unix)]
11713                    {
11714                        rustix::fs::accessat(
11715                            rustix::fs::CWD,
11716                            path,
11717                            rustix::fs::Access::EXEC_OK,
11718                            rustix::fs::AtFlags::EACCESS,
11719                        )
11720                        .is_ok()
11721                    }
11722                    #[cfg(not(unix))]
11723                    {
11724                        true
11725                    }
11726                })
11727                .ok_or_else(|| SuperviseError::Spawn {
11728                    program: spec.program.clone(),
11729                    source: io::Error::new(io::ErrorKind::NotFound, "executable not found in PATH"),
11730                    cgroup_path: None,
11731                })?
11732        };
11733        let resolved = std::fs::canonicalize(&path).map_err(|source| SuperviseError::Spawn {
11734            program: spec.program.clone(),
11735            source,
11736            cgroup_path: None,
11737        })?;
11738        before_open(&resolved);
11739        // Resolving links does not open their target. The descriptor open must
11740        // remain nonblocking and no-follow if that resolved name is substituted.
11741        let file =
11742            subc_os::open_regular_nofollow(&resolved).map_err(|source| SuperviseError::Spawn {
11743                program: spec.program.clone(),
11744                source: io::Error::other(source),
11745                cgroup_path: None,
11746            })?;
11747        let identity =
11748            subc_os::retained_file_identity(&file).map_err(|source| SuperviseError::Spawn {
11749                program: spec.program.clone(),
11750                source,
11751                cgroup_path: None,
11752            })?;
11753        Ok(Self {
11754            path,
11755            _file: file,
11756            identity,
11757        })
11758    }
11759    fn current(&self) -> bool {
11760        #[cfg(unix)]
11761        {
11762            subc_os::file_identity(&self.path) == Some(self.identity)
11763        }
11764        #[cfg(windows)]
11765        {
11766            std::fs::canonicalize(&self.path)
11767                .ok()
11768                .and_then(|path| subc_os::open_regular_nofollow(&path).ok())
11769                .and_then(|file| subc_os::retained_file_identity(&file).ok())
11770                == Some(self.identity)
11771        }
11772    }
11773}
11774
11775fn spawn_plan_stale(spec: &ModuleSpec) -> SuperviseError {
11776    SuperviseError::ConfigDeliveryDeferred {
11777        module_id: spec.module_id.clone(),
11778    }
11779}
11780
11781fn require_program_search_path(spec: &ModuleSpec) -> Result<(), SuperviseError> {
11782    if !cfg!(target_os = "macos")
11783        && !spec.program.is_absolute()
11784        && spec.program.components().count() == 1
11785        && !spec.env.iter().any(|(key, _)| {
11786            if cfg!(windows) {
11787                key.eq_ignore_ascii_case("PATH")
11788            } else {
11789                key == "PATH"
11790            }
11791        })
11792        && std::env::var_os("PATH").is_none()
11793    {
11794        // The OS default search is not a current-directory search. Refusing an
11795        // unspecified search path prevents approving a decoy in the current directory.
11796        return Err(SuperviseError::Spawn {
11797            program: spec.program.clone(),
11798            source: io::Error::new(io::ErrorKind::InvalidInput, "bare program requires PATH"),
11799            cgroup_path: None,
11800        });
11801    }
11802    Ok(())
11803}
11804
11805async fn spawn_child_async(
11806    spec: &ModuleSpec,
11807    runtime: &SupervisorRuntimeConfig,
11808) -> Result<SupervisedChild, SuperviseError> {
11809    #[cfg(all(test, unix))]
11810    if let Some(source) = containment_tests::join_failure(spec, &runtime.child_roster) {
11811        containment::terminate_blocking(&runtime.child_roster.containment);
11812        return Err(SuperviseError::Spawn {
11813            program: spec.program.clone(),
11814            source,
11815            cgroup_path: None,
11816        });
11817    }
11818    let result = spawn_child_in_slot_async(spec, runtime, SpawnRole::Plain, false).await;
11819    if result
11820        .as_ref()
11821        .is_err_and(|error| !is_launch_deferral(error))
11822    {
11823        if runtime.child_roster.admission_pending(&spec.module_id) {
11824            containment::terminate_deferred(&runtime.child_roster.containment);
11825        } else {
11826            containment::terminate_blocking(&runtime.child_roster.containment);
11827        }
11828    }
11829    result
11830}
11831
11832async fn spawn_child_in_slot_async(
11833    spec: &ModuleSpec,
11834    runtime: &SupervisorRuntimeConfig,
11835    role: SpawnRole,
11836    alternate_slot: bool,
11837) -> Result<SupervisedChild, SuperviseError> {
11838    let gate = runtime
11839        .config_delivery
11840        .as_ref()
11841        .map(|delivery| delivery.0.module_gate(&spec.module_id))
11842        .transpose()
11843        .map_err(|error| SuperviseError::InvalidSpec {
11844            reason: error.to_string(),
11845        })?;
11846    if gate.as_ref().is_some_and(|gate| gate.try_lock().is_err()) {
11847        return Err(SuperviseError::ConfigDeliveryDeferred {
11848            module_id: spec.module_id.clone(),
11849        });
11850    }
11851    let prepared_spec = spec.clone();
11852    let prepared_runtime = runtime.clone();
11853    let prepared = tokio::task::spawn_blocking(move || {
11854        let prepared = prepare_child_in_slot(
11855            &prepared_spec,
11856            prepared_runtime.connection_file_path.as_deref(),
11857            prepared_runtime.supervisor_handle.as_ref(),
11858            prepared_runtime.capture_logs_dir.as_deref(),
11859            &prepared_runtime.child_roster,
11860            prepared_runtime.spawn_delivery(),
11861            #[cfg(target_os = "linux")]
11862            prepared_runtime.cgroup_placement.as_ref(),
11863            role,
11864            alternate_slot,
11865            true,
11866        );
11867        prepared
11868    })
11869    .await
11870    .map_err(|_| SuperviseError::InvalidSpec {
11871        reason: "configuration launch preparation stopped".into(),
11872    })??;
11873    if let Some(registration) = &prepared.delivery_registration {
11874        if !registration.launch_current()? {
11875            return Err(spawn_plan_stale(spec));
11876        }
11877    }
11878    let _gate = match gate {
11879        Some(gate) => {
11880            Some(
11881                gate.try_lock_owned()
11882                    .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11883                        module_id: spec.module_id.clone(),
11884                    })?,
11885            )
11886        }
11887        None => None,
11888    };
11889    if let (Some(delivery), Some(admission)) = (runtime.spawn_delivery(), &prepared.spawn_admission)
11890    {
11891        delivery.check(spec, admission)?;
11892    }
11893    if prepared
11894        .spawn_admission
11895        .as_ref()
11896        .is_none_or(|admission| admission.launch.is_none())
11897        && prepared
11898            .program_binding
11899            .as_ref()
11900            .is_some_and(|binding| !binding.current())
11901    {
11902        return Err(spawn_plan_stale(spec));
11903    }
11904    // No await may separate this last identity check from ownership publication.
11905    let checked_base = prepared
11906        .spawn_admission
11907        .as_ref()
11908        .and_then(|admission| admission.approved_base())
11909        .cloned();
11910    let mut result = launch_child_in_slot(
11911        spec,
11912        &runtime.stderr_ring,
11913        &runtime.child_roster,
11914        prepared,
11915        runtime.register_timeout,
11916    );
11917    drop(_gate);
11918    if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11919        (runtime.config_delivery.as_ref(), &result)
11920    {
11921        if let Some(base) = checked_base.filter(|_| code == "program_refused") {
11922            let module = spec.module_id.clone();
11923            let identity = subc_config::raw_digest(detail.as_bytes());
11924            let recorded = delivery
11925                .0
11926                .on_blocking(move |owner| owner.record_program_refusal(&module, &base, identity))
11927                .await;
11928            match recorded {
11929                Ok(true) => {}
11930                Ok(false) => result = Err(spawn_plan_stale(spec)),
11931                Err(error) => {
11932                    result = Err(config_delivery::gate_hold_error(
11933                        spec,
11934                        error,
11935                        Some(delivery),
11936                    ))
11937                }
11938            }
11939        }
11940    }
11941    result
11942}
11943
11944fn validate_spawn_program(
11945    spec: &ModuleSpec,
11946    admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11947    delivery: Option<config_delivery::SpawnDelivery<'_>>,
11948) -> Result<(), SuperviseError> {
11949    if let Some(admission) = admission {
11950        if let Some(program) = &admission.launch {
11951            if let Err(error) = program.validate_for_spawn() {
11952                // A captured unsigned image is a definite refusal. Uncertain
11953                // reads are deferred by validate_for_spawn, not attributed to
11954                // the operator as a changed or unapprovable program.
11955                if let (Some(context), Some(base)) = (delivery, admission.approved_base()) {
11956                    if error.code == "program_refused"
11957                        && !context.record_refusal(spec, base, &error.detail)?
11958                    {
11959                        return Err(spawn_plan_stale(spec));
11960                    }
11961                }
11962                return Err(config_delivery::gate_hold_error(
11963                    spec,
11964                    error,
11965                    delivery.map(|context| context.runtime()).as_ref(),
11966                ));
11967            }
11968        }
11969    }
11970    Ok(())
11971}
11972
11973#[allow(clippy::too_many_arguments)]
11974fn prepare_child_in_slot(
11975    spec: &ModuleSpec,
11976    connection_file_path: Option<&std::path::Path>,
11977    handle: Option<&SupervisorHandle>,
11978    capture_logs_dir: Option<&std::path::Path>,
11979    roster: &ChildRoster,
11980    config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11981    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11982    role: SpawnRole,
11983    alternate_slot: bool,
11984    asynchronous: bool,
11985) -> Result<PreparedChild, SuperviseError> {
11986    require_program_search_path(spec)?;
11987    let spawn_admission = config_delivery
11988        .map(|delivery| delivery.prepare(spec))
11989        .transpose()?;
11990    #[cfg(target_os = "macos")]
11991    let gated = spawn_admission
11992        .as_ref()
11993        .is_some_and(|admission| admission.approved_base().is_some());
11994    let mut launch_spec = spawn_admission
11995        .as_ref()
11996        .and_then(|admission| admission.launch.as_ref())
11997        .map(|launch| launch.spec.clone())
11998        .unwrap_or_else(|| spec.clone());
11999    if let Some(script) = &launch_spec.script {
12000        launch_spec.program =
12001            script::runner_path(script.runtime).map_err(|source| SuperviseError::Spawn {
12002                program: launch_spec.program.clone(),
12003                source,
12004                cgroup_path: None,
12005            })?;
12006    }
12007    let script_content = if let Some(entry) = &launch_spec.script {
12008        Some(
12009            spawn_admission
12010                .as_ref()
12011                .and_then(|admission| admission.launch.as_ref())
12012                .and_then(|launch| launch.script.clone())
12013                .map_or_else(|| script::ScriptBytes::read(&entry.path), Ok)
12014                .map_err(|error| config_delivery::gate_hold_error(&launch_spec, error, None))?,
12015        )
12016    } else {
12017        None
12018    };
12019    let spec = &launch_spec;
12020    if let Some(error) = recovery_block_error(spec, roster) {
12021        return Err(error);
12022    }
12023    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
12024        return Err(previous_instance_error(spec, pid, reason));
12025    }
12026    if roster.is_closed() {
12027        return Err(SuperviseError::Spawn {
12028            program: spec.program.clone(),
12029            source: io::Error::other("the daemon is shutting down; not starting a new process"),
12030            cgroup_path: None,
12031        });
12032    }
12033    let carried = containment::current(&roster.containment);
12034    let carrying = carries_containment(spec.carry_containment, carried.is_some());
12035    if role != SpawnRole::Plain && carrying {
12036        return Err(SuperviseError::InvalidSpec {
12037            reason: "a carried container cannot overlap a swap candidate".to_owned(),
12038        });
12039    }
12040    #[cfg(unix)]
12041    let launch_identity = if carrying && carried.is_none() {
12042        Some(
12043            containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
12044                program: spec.program.clone(),
12045                source,
12046                cgroup_path: None,
12047            })?,
12048        )
12049    } else {
12050        None
12051    };
12052    #[cfg(target_os = "linux")]
12053    let cgroup_placement = match &carried {
12054        Some(container) => container.cgroup().map(|(placement, _)| placement),
12055        None => cgroup_placement,
12056    };
12057    #[cfg(unix)]
12058    let group_target = if let Some(container) = carried
12059        .as_ref()
12060        .filter(|container| container.group().is_some())
12061    {
12062        containment::GroupTarget::Pinned(Arc::clone(container))
12063    } else {
12064        #[cfg(target_os = "linux")]
12065        let cgroup_container = carrying && cgroup_placement.is_some();
12066        #[cfg(not(target_os = "linux"))]
12067        let cgroup_container = false;
12068        if cgroup_container {
12069            containment::GroupTarget::NoGroup
12070        } else {
12071            containment::GroupTarget::OwnLeader
12072        }
12073    };
12074    #[cfg(target_os = "linux")]
12075    let cgroup_name = {
12076        // Slot names alone are not kill domains: a retired incumbent may still
12077        // be draining when a later enable/restart spawns into the same slot.
12078        // Decimal entropy keeps the suffix unambiguous; Placement performs
12079        // the module-id escaping and constructs the filesystem path.
12080        if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
12081            name.to_owned()
12082        } else if cgroup_placement.is_none() {
12083            swap::cgroup_name(&spec.module_id, alternate_slot)
12084        } else {
12085            let nonce = generate_launch_nonce()?;
12086            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
12087            // Leave room for byte escaping and the suffix under NAME_MAX. The
12088            // label is only for humans; the nonce identifies the kill domain.
12089            let mut end = spec.module_id.len().min(64);
12090            while !spec.module_id.is_char_boundary(end) {
12091                end -= 1;
12092            }
12093            format!(
12094                "{}_{suffix}",
12095                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
12096            )
12097        }
12098    };
12099    #[cfg(not(target_os = "linux"))]
12100    let _ = alternate_slot;
12101    #[cfg(windows)]
12102    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
12103        program: spec.program.clone(),
12104        source,
12105        cgroup_path: None,
12106    })?;
12107    #[cfg(windows)]
12108    let program_binding = if asynchronous {
12109        let mut resolved = spec.clone();
12110        resolved.program = image_capture.path().to_path_buf();
12111        Some(SpawnProgramBinding::capture(&resolved)?)
12112    } else {
12113        None
12114    };
12115    #[cfg(not(windows))]
12116    let program_binding = (asynchronous || cfg!(target_os = "macos"))
12117        .then(|| SpawnProgramBinding::capture(spec))
12118        .transpose()?;
12119    #[cfg(target_os = "macos")]
12120    validate_spawn_program(spec, spawn_admission.as_ref(), config_delivery)?;
12121    #[cfg(target_os = "macos")]
12122    let (mut command, privacy_exec, exec_ack) = privacy_command(
12123        spec,
12124        program_binding
12125            .as_ref()
12126            .expect("macOS captures every launch"),
12127        roster,
12128        // Ungated launches retain legacy unsuspended exec, including scripts. Suspended
12129        // verification admits approved programs only. prepare() above fails
12130        // closed if the gate cannot decide; an image sample is not authority.
12131        gated,
12132    )?;
12133    #[cfg(windows)]
12134    let mut command = Command::new(image_capture.path());
12135    #[cfg(not(any(target_os = "macos", windows)))]
12136    let mut command = Command::new(
12137        program_binding
12138            .as_ref()
12139            .map_or(spec.program.as_path(), |binding| binding.path.as_path()),
12140    );
12141    command.args(&spec.args);
12142    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
12143    // that is the whole of the intent, so remove that one key rather than the
12144    // environment.
12145    //
12146    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
12147    // and took the POSIX environment with it. Modules spawned that way had no
12148    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
12149    // logging:
12150    //
12151    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
12152    //     both unset it fell back to the temp dir alone and `ck` could not find
12153    //     a daemon running on the same machine from inside any module's process
12154    //     tree — reporting a path the file has never lived at, which reads as
12155    //     "the daemon did not write its file".
12156    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
12157    //     the RELATIVE `.local/share`, so a module deriving its own store path
12158    //     resolved it against its own CWD. That is the store-fragmentation
12159    //     defect the daemon already refuses in config (`parse_doc` rejects a
12160    //     relative `storage.data_home`) arriving by derivation instead.
12161    //   * anything a module spawns inherited it: git without ~/.gitconfig,
12162    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
12163    //     quietly rather than erroring.
12164    //
12165    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
12166    // offered one candidate under /tmp while the file sat in /run/user/1000.
12167    //
12168    // A configured module is unaffected either way: `module_spec()` puts the
12169    // resolved CK_LOG into `spec.env`, which is applied below and therefore
12170    // wins over anything ambient.
12171    apply_child_env(&mut command, spec);
12172    if spec.script.is_some() {
12173        script::remove_runtime_env(&mut command, spec);
12174    }
12175    let origin_witness = crate::program_origin::LaunchWitness::capture_command(spec, &command)
12176        .map_err(|source| SuperviseError::Spawn {
12177            program: spec.program.clone(),
12178            source,
12179            cgroup_path: None,
12180        })?;
12181    command.current_dir(&origin_witness.working_directory);
12182    apply_spawn_role(&mut command, role);
12183    let nonce_handoff = apply_wire_spawn_args_for_admission(
12184        &mut command,
12185        spec,
12186        connection_file_path,
12187        handle,
12188        role,
12189        spawn_admission.as_ref(),
12190    )?;
12191    #[cfg(unix)]
12192    let mut slots = subc_os::registration_channel::ChildDescriptorSlots::default();
12193    command.env_remove(subc_os::script_pipe::SCRIPT_FD_ENV);
12194    let script_delivery = script_content
12195        .map(|content| {
12196            script::Delivery::prepare(
12197                content,
12198                &mut command,
12199                #[cfg(unix)]
12200                &mut slots,
12201            )
12202        })
12203        .transpose()
12204        .map_err(|source| SuperviseError::Spawn {
12205            program: spec.program.clone(),
12206            source,
12207            cgroup_path: None,
12208        })?;
12209    #[cfg(unix)]
12210    let registration_channel = if spec.protocol == ModuleProtocol::Subc {
12211        let (parent, child, metadata) = subc_os::registration_channel::registration_socketpair()
12212            .map_err(|source| SuperviseError::Spawn {
12213                program: spec.program.clone(),
12214                source,
12215                cgroup_path: None,
12216            })?;
12217        slots
12218            .channel(child)
12219            .map_err(|source| SuperviseError::Spawn {
12220                program: spec.program.clone(),
12221                source,
12222                cgroup_path: None,
12223            })?;
12224        command
12225            .env(subc_os::registration_channel::REGISTRATION_FD_ENV, metadata)
12226            .env(
12227                subc_os::registration_channel::DAEMON_PID_ENV,
12228                std::process::id().to_string(),
12229            );
12230        Some(parent)
12231    } else {
12232        None
12233    };
12234    #[cfg(windows)]
12235    let registration_channel = prepare_windows_registration_pipe(&mut command, spec)?;
12236    let binding_publication = if spawn_admission
12237        .as_ref()
12238        .is_some_and(|admission| admission.launch.is_some())
12239    {
12240        handle.and_then(|handle| handle.begin_program_binding(&spec.module_id, role))
12241    } else {
12242        None
12243    };
12244    let registration_spawn = (spec.protocol == ModuleProtocol::Subc)
12245        .then(|| {
12246            handle.map(|handle| {
12247                let spawn = RegistrationSpawn::new_with_binding_requirement(
12248                    handle,
12249                    &spec.module_id,
12250                    role,
12251                    binding_publication.clone(),
12252                    spawn_admission
12253                        .as_ref()
12254                        .is_some_and(|admission| admission.launch.is_some()),
12255                );
12256                if let Some(admission) = &spawn_admission {
12257                    admission.initialize_origin(&spawn.authority, &origin_witness);
12258                }
12259                handle.record_nonce_launch(&spawn);
12260                spawn
12261            })
12262        })
12263        .flatten();
12264
12265    #[cfg(target_os = "linux")]
12266    let cgroup_path = cgroup_placement
12267        .map(|placement| {
12268            if let Some(container) = &carried {
12269                let path = container
12270                    .cgroup_path()
12271                    .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
12272                placement.existing_module_path(path)?.ok_or_else(|| {
12273                    io::Error::new(
12274                        io::ErrorKind::NotFound,
12275                        "carried cgroup disappeared before replacement",
12276                    )
12277                })
12278            } else {
12279                if carrying {
12280                    containment::prepare_cgroup_path(placement, &cgroup_name)
12281                } else {
12282                    placement.module_path(&cgroup_name)
12283                }
12284            }
12285        })
12286        .transpose()
12287        .map_err(|source| SuperviseError::Cgroup {
12288            module_id: spec.module_id.clone(),
12289            source,
12290        })?;
12291    #[cfg(not(target_os = "linux"))]
12292    let cgroup_path: Option<PathBuf> = None;
12293    // Own a newly created directory before any fallible admission or command
12294    // preparation. Existing carried containment must never be removed here.
12295    #[cfg(target_os = "linux")]
12296    let cgroup_cleanup = PreparedCgroup(
12297        cgroup_placement
12298            .filter(|_| carried.is_none())
12299            .map(|placement| (placement.clone(), cgroup_name.clone())),
12300    );
12301    #[cfg(unix)]
12302    let mut admission_gate = if carrying {
12303        Some(
12304            admission::Gate::install(&mut command, spec, roster).map_err(|source| {
12305                SuperviseError::Spawn {
12306                    program: spec.program.clone(),
12307                    source,
12308                    cgroup_path: cgroup_path.clone(),
12309                }
12310            })?,
12311        )
12312    } else {
12313        None
12314    };
12315    #[cfg(target_os = "linux")]
12316    if let Some(path) = &cgroup_path {
12317        apply_cgroup_placement(&mut command, spec, path)?;
12318    }
12319
12320    let output_sink = if let Some(logs_dir) = capture_logs_dir {
12321        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
12322        match ChildOutputSink::open(&path, capture_retention(spec)) {
12323            Ok(sink) => sink,
12324            Err(error) => {
12325                warn!(
12326                    module_id = %spec.module_id,
12327                    path = %path.display(),
12328                    error = %error,
12329                    "could not open child output capture file; forwarding to stderr"
12330                );
12331                ChildOutputSink::Stderr
12332            }
12333        }
12334    } else {
12335        ChildOutputSink::Stderr
12336    };
12337
12338    command.stdout(Stdio::piped());
12339    command.stderr(Stdio::piped());
12340    command.kill_on_drop(!carrying);
12341    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
12342    // before exec). In the daemon's group, a service manager that kills the
12343    // job's process group when the daemon exits (launchd's default) killed
12344    // every module at the same moment its control connection closed, so no
12345    // module ever ran its EOF teardown on a daemon stop. Outside that group a
12346    // module is reached only by the daemon: the EOF it sees when its
12347    // connection closes, and the bounded stop in `child_roster` for anything
12348    // still running after that. On Linux this composes with the cgroup
12349    // placement above: that is a pre_exec write to cgroup.procs, std performs
12350    // setpgid in the child before running pre_exec callbacks, and the two
12351    // change independent process attributes.
12352    //
12353    // stdin is /dev/null because a process outside the terminal's foreground
12354    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
12355    // by hand would otherwise hand down. Under a service manager stdin is
12356    // already /dev/null.
12357    #[cfg(unix)]
12358    command.process_group(
12359        carried
12360            .as_ref()
12361            .map(|container| container.verified_join_group())
12362            .transpose()
12363            .map_err(|source| SuperviseError::Spawn {
12364                program: spec.program.clone(),
12365                source,
12366                cgroup_path: cgroup_path.clone(),
12367            })?
12368            .flatten()
12369            .unwrap_or(0) as i32,
12370    );
12371    command.stdin(Stdio::null());
12372    #[cfg(unix)]
12373    if let Some(gate) = &mut admission_gate {
12374        gate.install_placement(&mut command);
12375    }
12376    // Fill all reserved slots in one allocation-free step, after placement has
12377    // stopped using descriptors that the fixed slots may replace.
12378    #[cfg(unix)]
12379    {
12380        if let Some(handoff) = nonce_handoff {
12381            slots.nonce(handoff);
12382        }
12383        #[cfg(target_os = "macos")]
12384        slots
12385            .acknowledgement(exec_ack.into_descriptor())
12386            .map_err(|source| SuperviseError::Spawn {
12387                program: spec.program.clone(),
12388                source,
12389                cgroup_path: cgroup_path.clone(),
12390            })?;
12391        slots.install(command.as_std_mut());
12392    }
12393    #[cfg(not(any(unix, windows)))]
12394    let _ = nonce_handoff;
12395
12396    // Containment, step 1 of 3 (issue #109): create the child suspended so it
12397    // cannot run a single instruction -- and therefore cannot spawn a
12398    // grandchild -- before it is in the job. See `contain_spawned_child` for the
12399    // other two steps and why the window matters.
12400    #[cfg(windows)]
12401    {
12402        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
12403            program: spec.program.clone(),
12404            source,
12405            cgroup_path: None,
12406        })?;
12407        subc_jobobject::suspend_on_create_async(&mut command);
12408    }
12409    // Timestamp before the spawn call: on Unix the child can serve before that
12410    // call even returns. This conservative bound never labels an older child
12411    // as a replacement for a request received during process creation.
12412    let spawned_at = Instant::now();
12413    let delivery_context = config_delivery;
12414    let config_delivery = config_delivery
12415        .map(|delivery| {
12416            delivery.spawned(
12417                spec,
12418                spawn_admission
12419                    .as_ref()
12420                    .expect("delivery prepares spawn admission"),
12421                spawned_at,
12422                role,
12423                registration_spawn
12424                    .as_ref()
12425                    .map(|spawn| spawn.authority.clone()),
12426            )
12427        })
12428        .transpose()?
12429        .flatten();
12430    let mut delivery_registration =
12431        delivery_context.map(|context| context.registration_guard(config_delivery.as_ref(), role));
12432    if let Some(registration) = &mut delivery_registration {
12433        registration.defer_cleanup();
12434    }
12435    if asynchronous {
12436        #[cfg(test)]
12437        if let Some(context) = delivery_context {
12438            context.pause_preflight_for_test();
12439        }
12440    }
12441    validate_spawn_program(spec, spawn_admission.as_ref(), delivery_context)?;
12442    Ok(PreparedChild {
12443        script_delivery,
12444        registration_spawn,
12445        registration_handle: handle.cloned(),
12446        #[cfg(any(unix, windows))]
12447        registration_channel,
12448        config_context: delivery_context.map(|delivery| delivery.runtime()),
12449        binding_publication,
12450        launch_spec,
12451        spawn_admission,
12452        command,
12453        carrying,
12454        carried: carried.clone(),
12455        #[cfg(unix)]
12456        launch_identity,
12457        #[cfg(unix)]
12458        group_target,
12459        #[cfg(target_os = "linux")]
12460        cgroup_placement: cgroup_placement.cloned(),
12461        #[cfg(target_os = "linux")]
12462        cgroup_cleanup,
12463        #[cfg(target_os = "linux")]
12464        cgroup_name,
12465        cgroup_path,
12466        #[cfg(unix)]
12467        admission_gate,
12468        output_sink,
12469        #[cfg(target_os = "macos")]
12470        privacy_exec,
12471        #[cfg(windows)]
12472        image_capture,
12473        #[cfg(windows)]
12474        nonce_handoff,
12475        spawned_at,
12476        config_delivery,
12477        delivery_registration,
12478        program_binding,
12479    })
12480}
12481
12482fn launch_child_in_slot(
12483    _spec: &ModuleSpec,
12484    ring: &Arc<Mutex<StderrRing>>,
12485    roster: &ChildRoster,
12486    prepared: PreparedChild,
12487    register_timeout: Duration,
12488) -> Result<SupervisedChild, SuperviseError> {
12489    let launch_spec = prepared.launch_spec.clone();
12490    let registration_declaration = prepared
12491        .spawn_admission
12492        .as_ref()
12493        .and_then(|admission| admission.registration_declaration(&launch_spec.module_id));
12494    let admission_base = prepared
12495        .spawn_admission
12496        .as_ref()
12497        .and_then(|admission| admission.approved_base())
12498        .cloned();
12499    let spec = &launch_spec;
12500    let expected_image = prepared
12501        .spawn_admission
12502        .as_ref()
12503        .and_then(|admission| admission.launch.as_ref())
12504        .map(|launch| launch.image);
12505    let program_launch = prepared
12506        .spawn_admission
12507        .and_then(|admission| admission.launch);
12508    let config_owner = prepared
12509        .config_context
12510        .as_ref()
12511        .map(|context| context.0.clone());
12512    let binding_publication = prepared.binding_publication;
12513    #[cfg(test)]
12514    if let Some(context) = &prepared.config_context {
12515        context.0.before_spawn_exec_for_test();
12516    }
12517    #[cfg(feature = "test-support")]
12518    if let Some(context) = &prepared.config_context {
12519        context
12520            .0
12521            .pause_spawn_before_exec()
12522            .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(context)))?;
12523    }
12524    let mut command = prepared.command;
12525    let carrying = prepared.carrying;
12526    let carried = prepared.carried;
12527    #[cfg(unix)]
12528    let launch_identity = prepared.launch_identity;
12529    #[cfg(unix)]
12530    let group_target = prepared.group_target;
12531    #[cfg(target_os = "linux")]
12532    let cgroup_placement = prepared.cgroup_placement.as_ref();
12533    #[cfg(target_os = "linux")]
12534    let cgroup_name = prepared.cgroup_name;
12535    #[cfg(target_os = "linux")]
12536    let mut cgroup_cleanup = prepared.cgroup_cleanup;
12537    #[cfg(target_os = "linux")]
12538    let _ = cgroup_cleanup.0.take();
12539    let cgroup_path = prepared.cgroup_path;
12540    #[cfg(unix)]
12541    let admission_gate = prepared.admission_gate;
12542    let output_sink = prepared.output_sink;
12543    #[cfg(target_os = "macos")]
12544    let mut privacy_exec = prepared.privacy_exec;
12545    #[cfg(target_os = "macos")]
12546    if let Some(pending) = &mut privacy_exec {
12547        pending.admission_binding = binding_publication.clone();
12548        if expected_image.is_some() {
12549            pending.expected = expected_image;
12550        }
12551    }
12552    #[cfg(any(unix, windows))]
12553    let registration_channel = prepared.registration_channel;
12554    let registration_spawn = prepared.registration_spawn;
12555    let registration_handle = prepared.registration_handle;
12556    #[cfg(windows)]
12557    let image_capture = prepared.image_capture;
12558    #[cfg(windows)]
12559    let nonce_handoff = prepared.nonce_handoff;
12560    let spawned_at = prepared.spawned_at;
12561    let config_delivery = prepared.config_delivery;
12562    let delivery_registration = prepared.delivery_registration;
12563    let _program_binding = prepared.program_binding;
12564    #[cfg(all(test, unix))]
12565    containment_tests::record_spawn_attempt(spec, &mut command);
12566    #[cfg(unix)]
12567    let spawned = match admission_gate {
12568        Some(gate) => gate
12569            .spawn(
12570                command,
12571                spec,
12572                roster,
12573                launch_identity,
12574                #[cfg(target_os = "linux")]
12575                cgroup_placement
12576                    .zip(cgroup_path.as_ref())
12577                    .map(|(placement, path)| {
12578                        (placement.clone(), cgroup_name.clone(), path.clone())
12579                    }),
12580            )
12581            .map(|(child, admitted)| (child, Some(admitted))),
12582        None => {
12583            let spawned =
12584                subc_os::child_spawn::tokio_spawn(&mut command).map(|child| (child, None));
12585            // The callback owns all child-only sources. Closing them here is
12586            // necessary for channel EOF and macOS exec acknowledgement EOF.
12587            drop(command);
12588            spawned
12589        }
12590    };
12591    #[cfg(not(unix))]
12592    let spawned = subc_os::child_spawn::tokio_spawn(&mut command);
12593    let spawned = match spawned {
12594        Ok(child) => child,
12595        Err(source) => {
12596            if carried
12597                .as_ref()
12598                .and_then(|container| container.group())
12599                .is_some()
12600            {
12601                containment::terminate_blocking(&roster.containment);
12602            }
12603            #[cfg(target_os = "linux")]
12604            if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12605                remove_module_cgroup(placement, &cgroup_name);
12606            }
12607            return Err(SuperviseError::Spawn {
12608                program: spec.program.clone(),
12609                source,
12610                cgroup_path,
12611            });
12612        }
12613    };
12614    #[cfg(unix)]
12615    let (mut child, mut admitted) = spawned;
12616    #[cfg(unix)]
12617    let group_target = admitted
12618        .as_ref()
12619        .map(|attempt| attempt.group_target())
12620        .unwrap_or(group_target);
12621    #[cfg(not(unix))]
12622    let mut child = spawned;
12623    if let Some(delivery) = prepared.script_delivery {
12624        delivery.start(child.id().expect("spawned child has a pid"));
12625    }
12626    #[cfg(test)]
12627    spawn_record_tests::after_create(&child, spec);
12628
12629    // Bind while the child is still suspended, before job assignment resumes it.
12630    // Keeping the executable handle open across CreateProcessW denies rename
12631    // and deletion, so the file cannot be exchanged during spawn. Bind that
12632    // file ID to the process handle returned by spawn, never a reopened PID.
12633    #[cfg(windows)]
12634    let windows_image = match image_capture.bind_tokio(&child) {
12635        Ok(image) => Arc::new(image),
12636        Err(source) => {
12637            kill_and_reap_rejected_child(&mut child);
12638            return Err(SuperviseError::Spawn {
12639                program: spec.program.clone(),
12640                source,
12641                cgroup_path: None,
12642            });
12643        }
12644    };
12645    // Assign the suspended child to its module job; resume only after its
12646    // process identity has been durably recorded for orphan recovery.
12647    #[cfg(windows)]
12648    let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
12649    let spawned_at_ms = unix_ms_now();
12650    #[cfg(not(windows))]
12651    let spawned_from = spec.program.clone();
12652    #[cfg(windows)]
12653    let spawned_from = windows_image.path().to_path_buf();
12654    #[cfg(not(windows))]
12655    let spawned_file_identity = spawned_file_identity(&spawned_from);
12656    #[cfg(windows)]
12657    let spawned_file_identity = Some(SpawnedFileIdentity {
12658        device: 0,
12659        inode: 0,
12660        windows_image: Some(Arc::clone(&windows_image)),
12661    });
12662    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
12663        program: spec.program.clone(),
12664        source: io::Error::other("spawned child exposed no live pid"),
12665        cgroup_path: cgroup_path.clone(),
12666    })?;
12667    #[cfg(windows)]
12668    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
12669    #[cfg(not(windows))]
12670    let process_start_time = admitted
12671        .as_ref()
12672        .map(|admitted| admitted.birth)
12673        .or_else(|| crate::provenance::process_start_time(pid));
12674    #[cfg(windows)]
12675    let process_start_time = Some(windows_image.start_time());
12676    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
12677    #[cfg(all(test, target_os = "macos"))]
12678    privacy_exec_boundary_tests::before_image_sample(spec, pid);
12679    #[cfg(all(test, target_os = "linux"))]
12680    assert!(roster.admission_event_for_test(&spec.module_id, "before_image_binding", Some(pid)));
12681    // Record the committed kernel image, not the configured pathname: PATH
12682    // lookup and shebang interpretation can select a different executable.
12683    // Linux's posix_spawn can return before that image is installed, so the
12684    // observer also waits for the kernel's first-exec flag to clear.
12685    #[cfg(not(windows))]
12686    let recorded_image = observe_spawned_image(pid);
12687    #[cfg(all(test, target_os = "linux"))]
12688    let recorded_image = if roster.admission_event_for_test(
12689        &spec.module_id,
12690        "after_spawn_image_sample",
12691        Some(pid),
12692    ) {
12693        recorded_image
12694    } else {
12695        None
12696    };
12697    #[cfg(target_os = "linux")]
12698    // An exited child cannot receive HELLO service and has no readable /proc/exe.
12699    // Keep its real exit for the supervisor instead of turning a crash into a hold.
12700    let exited_before_binding = expected_image.is_some()
12701        && recorded_image.and_then(|image| image.executable) != expected_image
12702        && child.try_wait().ok().flatten().is_some();
12703    #[cfg(target_os = "linux")]
12704    let unavailable_initial_image = expected_image.is_some()
12705        && recorded_image.and_then(|image| image.executable).is_none()
12706        && !exited_before_binding;
12707    #[cfg(not(target_os = "linux"))]
12708    let unavailable_initial_image = false;
12709    #[cfg(target_os = "linux")]
12710    if expected_image
12711        .is_some_and(|expected| recorded_image.and_then(|image| image.executable) != Some(expected))
12712        && !exited_before_binding
12713        && !unavailable_initial_image
12714    {
12715        kill_module_cgroup(cgroup_placement, &cgroup_name);
12716        kill_and_reap_rejected_child(&mut child, group_target.clone());
12717        return Err(SuperviseError::ConfigHeld {
12718            module_id: spec.module_id.clone(),
12719            code: "program_refused".into(),
12720            detail: format!(
12721                "spawned executable {:?} does not match hashed program {:?}",
12722                recorded_image.and_then(|image| image.executable),
12723                expected_image
12724            ),
12725        });
12726    }
12727    #[cfg(not(target_os = "linux"))]
12728    let _ = expected_image;
12729    #[cfg(target_os = "macos")]
12730    let binding_ready = privacy_exec.is_none();
12731    #[cfg(target_os = "linux")]
12732    let binding_ready = !exited_before_binding;
12733    #[cfg(not(any(target_os = "macos", target_os = "linux")))]
12734    let binding_ready = cfg!(windows);
12735    // spawn() confirms only the first exec, into the trampoline. Never persist
12736    // the trampoline image; the asynchronous acknowledgement publishes the
12737    // module image once the trampoline has replaced itself with the module.
12738    #[cfg(target_os = "macos")]
12739    let recorded_image = if privacy_exec.is_some() {
12740        None
12741    } else {
12742        recorded_image
12743    };
12744    #[cfg(target_os = "linux")]
12745    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
12746    #[cfg(not(target_os = "linux"))]
12747    let recorded_cgroup_name = None;
12748    #[cfg(unix)]
12749    let preadmitted = admitted
12750        .as_mut()
12751        .and_then(|admitted| admitted.leader.take());
12752    #[cfg(not(unix))]
12753    let preadmitted: Option<crate::child_roster::RosterGuard> = None;
12754    let roster_guard = if let Some(guard) = preadmitted {
12755        Ok(guard)
12756    } else {
12757        roster.admit(
12758            spec.module_id.clone(),
12759            pid,
12760            spec.protocol,
12761            process_start_time,
12762            crate::child_roster::RecordedIdentity {
12763                container: None,
12764                #[cfg(unix)]
12765                carried: None,
12766                #[cfg(not(windows))]
12767                start_time: recorded_image.map(|image| image.start_time),
12768                #[cfg(windows)]
12769                start_time: process_start_time,
12770                #[cfg(not(windows))]
12771                executable: recorded_image
12772                    .and_then(|image| image.executable)
12773                    .map(crate::live_children::ExecutableIdentity::from),
12774                #[cfg(windows)]
12775                executable: Some(crate::live_children::ExecutableIdentity::from(
12776                    windows_image.identity(),
12777                )),
12778                cgroup_name: recorded_cgroup_name,
12779                #[cfg(target_os = "linux")]
12780                cgroup_placement: cgroup_placement.cloned(),
12781            },
12782        )
12783    };
12784    let roster_guard = match roster_guard {
12785        Ok(guard) => guard,
12786        Err(failure) => {
12787            return failure.with_cleanup(|source| {
12788                #[cfg(windows)]
12789                if let Some(job) = &job {
12790                    let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12791                }
12792                #[cfg(target_os = "linux")]
12793                kill_module_cgroup(cgroup_placement, &cgroup_name);
12794                #[cfg(unix)]
12795                let rejected_group = group_target.clone();
12796                kill_and_reap_rejected_child(
12797                    &mut child,
12798                    #[cfg(unix)]
12799                    rejected_group,
12800                );
12801                #[cfg(target_os = "linux")]
12802                if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12803                    remove_module_cgroup(placement, &cgroup_name);
12804                }
12805                Err(SuperviseError::Spawn {
12806                    program: spec.program.clone(),
12807                    source: io::Error::other(format!(
12808                        "could not durably record module child: {source}"
12809                    )),
12810                    cgroup_path,
12811                })
12812            });
12813        }
12814    };
12815    #[cfg(unix)]
12816    if admitted.is_some() {
12817        if let Some(image) = recorded_image {
12818            if let Err(source) = roster_guard.confirm_executable(image) {
12819                kill_and_reap_rejected_child(&mut child, group_target.clone());
12820                return Err(SuperviseError::Spawn {
12821                    program: spec.program.clone(),
12822                    source,
12823                    cgroup_path,
12824                });
12825            }
12826        }
12827    }
12828    // The pre-spawn check can pass just before shutdown begins. Ownership is
12829    // published at the start of admission, even while a record write is queued.
12830    // If shutdown already saw an empty roster before that publication, this
12831    // check sees the roster closed and ends the child before starting a module.
12832    if roster.is_closed() {
12833        if carrying {
12834            #[cfg(unix)]
12835            let rejected_group = admitted
12836                .as_ref()
12837                .map(|admitted| admitted.rejection_target())
12838                .unwrap_or_else(|| group_target.clone());
12839            #[cfg(windows)]
12840            if let Some(job) = &job {
12841                let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12842            }
12843            kill_and_reap_rejected_child(
12844                &mut child,
12845                #[cfg(unix)]
12846                rejected_group,
12847            );
12848            #[cfg(all(test, unix))]
12849            let _ =
12850                roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
12851            drop(roster_guard);
12852            return Err(SuperviseError::Spawn {
12853                program: spec.program.clone(),
12854                source: io::Error::other(
12855                    "the daemon began shutting down during carried admission; ended the container",
12856                ),
12857                cgroup_path,
12858            });
12859        }
12860        // This child was never admitted, so there is no module protocol shutdown to wait for.
12861        #[cfg(target_os = "linux")]
12862        kill_module_cgroup(cgroup_placement, &cgroup_name);
12863        if let Err(error) = child.start_kill() {
12864            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
12865        }
12866        #[cfg(target_os = "linux")]
12867        if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12868            // This spawn was never admitted, so shutdown has no roster entry
12869            // to await. Do not detach its cleanup: the runtime could exit
12870            // before that task reaps the rejected child and removes its group.
12871            while matches!(child.try_wait(), Ok(None)) {
12872                std::thread::yield_now();
12873            }
12874            if matches!(
12875                subc_cgroup::kill_module(Some(placement), &cgroup_name),
12876                subc_cgroup::KillOutcome::Killed
12877            ) {
12878                if let Ok(path) = placement.module_path(&cgroup_name) {
12879                    while std::fs::read_to_string(path.join("cgroup.events"))
12880                        .ok()
12881                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
12882                    {
12883                        std::thread::yield_now();
12884                    }
12885                }
12886            }
12887            remove_module_cgroup(placement, &cgroup_name);
12888        }
12889        drop(roster_guard);
12890        return Err(SuperviseError::Spawn {
12891            program: spec.program.clone(),
12892            source: io::Error::other(
12893                "the daemon began shutting down while this process was starting; ended it",
12894            ),
12895            cgroup_path,
12896        });
12897    }
12898
12899    let stdout_pump = match child.stdout.take() {
12900        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
12901        None => {
12902            warn!(
12903                module_id = %spec.module_id,
12904                "spawned child exposed no stdout pipe; file capture will be incomplete"
12905            );
12906            None
12907        }
12908    };
12909    let stderr_pump = match child.stderr.take() {
12910        Some(stderr) => {
12911            let generation = ring
12912                .lock()
12913                .unwrap_or_else(|poisoned| poisoned.into_inner())
12914                .begin_process();
12915            Some(StderrPump {
12916                task: tokio::spawn(pump_stderr_to(
12917                    stderr,
12918                    Arc::clone(ring),
12919                    generation,
12920                    output_sink,
12921                )),
12922                generation,
12923            })
12924        }
12925        None => {
12926            // Spawning succeeded but the pipe did not materialise. Recording it as
12927            // uncaptured keeps the tail honest: the alternative is an empty tail
12928            // that reads as a module which printed nothing.
12929            ring.lock()
12930                .unwrap_or_else(|poisoned| poisoned.into_inner())
12931                .mark_not_captured("stderr pipe was not available on spawn");
12932            warn!(
12933                module_id = %spec.module_id,
12934                "spawned child exposed no stderr pipe; tail will be unavailable"
12935            );
12936            None
12937        }
12938    };
12939
12940    let exec = Arc::new(crate::activity::ExecOutcome::new());
12941    #[cfg(target_os = "macos")]
12942    let pending_exec = privacy_exec.is_some();
12943    #[cfg(not(target_os = "macos"))]
12944    let pending_exec = false;
12945    if !pending_exec {
12946        exec.finish(crate::activity::ExecState::CommittedExec);
12947    }
12948    #[cfg(windows)]
12949    let subtree_target = job
12950        .as_ref()
12951        .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
12952        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
12953    #[cfg(target_os = "linux")]
12954    let subtree_target = cgroup_path
12955        .map(crate::subtree_resources::Target::Cgroup)
12956        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
12957    #[cfg(not(any(target_os = "linux", windows)))]
12958    let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
12959    let mut supervised = SupervisedChild {
12960        registration_spawn,
12961        #[cfg(any(unix, windows))]
12962        registration_channel,
12963        live_program_binding: None,
12964        registered_config_connection: None,
12965        admission_base,
12966        registration_declaration,
12967        _program_binding: binding_publication,
12968        subtree_target,
12969        child: containment::ProcessChild::plain(child, pid),
12970        exec: crate::activity::ExecOwner(exec),
12971        #[cfg(windows)]
12972        nonce_delivery,
12973        protocol: spec.protocol,
12974        #[cfg(target_os = "linux")]
12975        module_id: cgroup_name,
12976        #[cfg(target_os = "linux")]
12977        cgroup_placement: cgroup_placement.cloned(),
12978        #[cfg(windows)]
12979        job,
12980        stdout_pump,
12981        stderr_pump,
12982        stderr_ring: Arc::clone(ring),
12983        spawned_at,
12984        config_delivery,
12985        registration_deadline: None,
12986        registration_watch: None,
12987        spawned_at_ms,
12988        spawned_from,
12989        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
12990        spawned_file_identity,
12991        process_start_time,
12992        process_identity,
12993        pid,
12994        roster_guard: Some(roster_guard),
12995        #[cfg(target_os = "macos")]
12996        privacy_exec,
12997        #[cfg(target_os = "macos")]
12998        report_ready: Arc::new(OnceLock::new()),
12999        spawn_failure: None,
13000        admission_kill_sent: false,
13001    };
13002    if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
13003        supervised.child.reject(
13004            #[cfg(unix)]
13005            group_target,
13006        );
13007        #[cfg(all(test, unix))]
13008        let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
13009        return Err(error);
13010    }
13011    if let (Some(launch), Some(owner), Some(base)) = (
13012        program_launch,
13013        config_owner,
13014        supervised.admission_base.clone(),
13015    ) {
13016        supervised.live_program_binding =
13017            content_binding::LiveProgramBinding::new(launch, &supervised, owner, base);
13018        if let (Some(publication), Some(live)) = (
13019            &supervised._program_binding,
13020            &supervised.live_program_binding,
13021        ) {
13022            publication.attach(live);
13023        }
13024    }
13025    #[cfg(unix)]
13026    if let Some(admitted) = admitted.take() {
13027        admitted.disarm();
13028    }
13029    #[cfg(windows)]
13030    if let Err(error) = resume_suspended_child(pid, spec) {
13031        containment::terminate_blocking(&roster.containment);
13032        let _ = supervised.child.start_kill();
13033        return Err(error);
13034    }
13035    if binding_ready {
13036        // An unreadable initial image is not a definite mismatch. Keep the
13037        // owned child and delivery registration until its reaped status decides
13038        // whether it crashed on its own or was stopped by admission's kill.
13039        let checked = if unavailable_initial_image {
13040            Err(crate::config_gate::GateError::new(
13041                "config_delivery_deferred",
13042                "initial process image is unavailable",
13043            ))
13044        } else {
13045            match (
13046                &supervised._program_binding,
13047                &supervised.live_program_binding,
13048            ) {
13049                (Some(binding), _) => binding.verify(),
13050                (None, Some(live)) => live.check(),
13051                _ if supervised.admission_base.is_some() => {
13052                    Err(crate::config_gate::GateError::new(
13053                        "config_delivery_deferred",
13054                        "live program witness is unavailable",
13055                    ))
13056                }
13057                _ => Ok(()),
13058            }
13059        };
13060        #[cfg(all(test, target_os = "linux"))]
13061        let checked = if roster.admission_event_for_test(
13062            &spec.module_id,
13063            "after_content_witness_check",
13064            Some(pid),
13065        ) {
13066            checked
13067        } else {
13068            Err(crate::config_gate::GateError::new(
13069                "config_delivery_deferred",
13070                "current process image is unavailable",
13071            ))
13072        };
13073        if let Err(error) = checked {
13074            if let Some(binding) = &mut supervised._program_binding {
13075                binding.reject();
13076            }
13077            // Reap through the normal child-reaping path, retaining the real exit record and
13078            // its roster ownership. A refusal must also end any helpers.
13079            if error.code == "program_refused" || supervised.try_wait().ok().flatten().is_none() {
13080                supervised.spawn_failure = Some(format!("{}: {}", error.code, error.detail));
13081                #[cfg(all(test, target_os = "linux"))]
13082                assert!(roster.admission_event_for_test(
13083                    &spec.module_id,
13084                    if unavailable_initial_image {
13085                        "before_initial_image_kill"
13086                    } else {
13087                        "before_content_witness_kill"
13088                    },
13089                    Some(pid),
13090                ));
13091                let _ = supervised.start_admission_kill();
13092                #[cfg(all(test, target_os = "linux"))]
13093                if supervised.admission_kill_sent {
13094                    assert!(roster.admission_event_for_test(
13095                        &spec.module_id,
13096                        "after_admission_kill_sent",
13097                        Some(pid),
13098                    ));
13099                }
13100            }
13101        }
13102    }
13103    if let Some(registration) = delivery_registration {
13104        registration.commit();
13105    }
13106    if let Some(spawn) = &supervised.registration_spawn {
13107        if let Some(identity) = supervised.process_identity {
13108            let _ = spawn.identity.set(identity);
13109        }
13110        if supervised.spawn_failure.is_some() {
13111            spawn.refuse();
13112        } else if binding_ready {
13113            spawn.ready();
13114        }
13115        #[cfg(any(unix, windows))]
13116        if let (Some(handle), Some(channel)) =
13117            (registration_handle, supervised.registration_channel.take())
13118        {
13119            handle.serve_registration(channel, spawn.clone(), pid, spawned_at + register_timeout);
13120        }
13121    }
13122    Ok(supervised)
13123}
13124
13125#[cfg(test)]
13126mod spawn_record_tests {
13127    use super::*;
13128
13129    thread_local! {
13130        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
13131    }
13132
13133    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
13134        LAST_PID.set(child.id());
13135        #[cfg(windows)]
13136        if let Some((_, address)) = _spec
13137            .env
13138            .iter()
13139            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
13140        {
13141            use std::io::{Read, Write};
13142            let mut stream = std::net::TcpStream::connect(address).unwrap();
13143            stream
13144                .write_all(&child.id().unwrap().to_ne_bytes())
13145                .unwrap();
13146            let mut release = [0];
13147            stream.read_exact(&mut release).unwrap();
13148        }
13149    }
13150
13151    #[cfg(windows)]
13152    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
13153        if spec
13154            .env
13155            .iter()
13156            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
13157        {
13158            Err(io::Error::other("injected module job assignment failure"))
13159        } else {
13160            Ok(())
13161        }
13162    }
13163
13164    fn spec() -> ModuleSpec {
13165        ModuleSpec {
13166            script: None,
13167            module_id: "spawn-record-worker".into(),
13168            #[cfg(unix)]
13169            program: "/bin/sleep".into(),
13170            #[cfg(windows)]
13171            program: std::env::current_exe().unwrap(),
13172            #[cfg(unix)]
13173            args: vec!["60".into()],
13174            #[cfg(windows)]
13175            args: vec![
13176                "--exact".into(),
13177                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
13178                "--nocapture".into(),
13179            ],
13180            env: vec![],
13181            reserved: false,
13182            reserved_prefixes: vec![],
13183            protocol: ModuleProtocol::None,
13184            overlap: Default::default(),
13185            carry_containment: false,
13186        }
13187    }
13188
13189    #[tokio::test]
13190    async fn record_sync_failure_fails_spawn_and_reaps_child() {
13191        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
13192        let supervisor =
13193            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
13194                .with_live_children_record(home.join("live-children.json"));
13195        crate::live_children::fail_next_file_sync();
13196        let result = supervisor.spawn(spec());
13197        let failed = result.is_err();
13198        if let Ok(module) = result {
13199            module.stop().await.unwrap();
13200        }
13201        assert!(failed, "record failure must fail spawn");
13202        let pid = LAST_PID.get().expect("the real OS child was created");
13203        assert!(
13204            subc_os::Process::open(pid).unwrap().is_none(),
13205            "child {pid} survived rejected admission or was not reaped"
13206        );
13207    }
13208
13209    #[cfg(target_os = "macos")]
13210    #[test]
13211    #[ignore = "manual on-device latency measurement"]
13212    fn durable_record_write_latency_50() {
13213        let home = cortexkit_test_support::ScratchDir::new("record-latency");
13214        let path = home.join("live-children.json");
13215        let children = vec![crate::live_children::LiveChild {
13216            module_id: "latency-worker".into(),
13217            pid: std::process::id(),
13218            protocol: ModuleProtocol::None,
13219            start_time: Some(1),
13220            executable: None,
13221            cgroup_name: None,
13222            container: None,
13223        }];
13224        let mut samples = Vec::new();
13225        for _ in 0..50 {
13226            let start = std::time::Instant::now();
13227            crate::live_children::write_record(&path, &children).unwrap();
13228            samples.push(start.elapsed());
13229        }
13230        samples.sort();
13231        eprintln!(
13232            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
13233            (samples[24] + samples[25]) / 2,
13234            samples[47]
13235        );
13236    }
13237
13238    #[cfg(windows)]
13239    #[test]
13240    fn windows_store_touch_fixture() {
13241        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
13242            std::fs::write(path, b"module ran").unwrap();
13243        }
13244    }
13245
13246    #[cfg(windows)]
13247    #[tokio::test]
13248    async fn windows_daemon_fixture() {
13249        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
13250            return;
13251        };
13252        let mut spec = spec();
13253        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
13254        let supervisor =
13255            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13256        let _module = supervisor.spawn(spec).unwrap();
13257    }
13258
13259    #[cfg(windows)]
13260    #[test]
13261    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
13262        use std::io::Read;
13263        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13264        // An outer job models a scheduler/service host. The daemon's root and
13265        // root job must nest without requiring breakaway from this job.
13266        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13267        command
13268            .args([
13269                "--exact",
13270                "supervise::spawn_record_tests::windows_daemon_fixture",
13271                "--nocapture",
13272            ])
13273            .env(
13274                "SUBC_TEST_DAEMON_BARRIER",
13275                listener.local_addr().unwrap().to_string(),
13276            );
13277        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13278        listener.set_nonblocking(true).unwrap();
13279        let deadline = std::time::Instant::now() + Duration::from_secs(30);
13280        let mut stream = loop {
13281            if let Ok((stream, _)) = listener.accept() {
13282                break stream;
13283            }
13284            assert!(
13285                daemon.child.try_wait().unwrap().is_none(),
13286                "daemon failed before rendezvous"
13287            );
13288            assert!(
13289                std::time::Instant::now() < deadline,
13290                "daemon did not reach CreateProcess failpoint"
13291            );
13292            std::thread::yield_now();
13293        };
13294        stream
13295            .set_read_timeout(Some(Duration::from_secs(30)))
13296            .unwrap();
13297        let mut pid = [0; 4];
13298        stream.read_exact(&mut pid).unwrap();
13299        let pid = u32::from_ne_bytes(pid);
13300        assert!(subc_jobobject::process_exists(pid));
13301        // Terminate only the daemon, not its outer job: otherwise the host job
13302        // would also kill a child whose daemon never joined its root job.
13303        daemon.child.kill().unwrap();
13304        daemon.child.wait().unwrap();
13305        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
13306        if !exited {
13307            daemon.job.terminate().unwrap();
13308        }
13309        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
13310    }
13311
13312    #[cfg(windows)]
13313    #[tokio::test]
13314    async fn windows_containment_failure_never_resumes() {
13315        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
13316        let sentinel = home.join("store-touched");
13317        let mut spec = spec();
13318        spec.env
13319            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
13320        spec.env.push((
13321            "SUBC_TEST_STORE_TOUCH".into(),
13322            sentinel.display().to_string(),
13323        ));
13324        let supervisor =
13325            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13326        let result = supervisor.spawn(spec);
13327        // A version that resumes despite assignment failure must have time to
13328        // execute its sentinel before dropping the returned module kills it.
13329        if let Ok(module) = result {
13330            let _ = tokio::time::timeout(Duration::from_secs(10), async {
13331                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
13332                    tokio::task::yield_now().await;
13333                }
13334            })
13335            .await;
13336            let _ = module.stop().await;
13337            panic!(
13338                "assignment failure resumed a module; store touched={}",
13339                sentinel.exists()
13340            );
13341        }
13342        assert!(
13343            !sentinel.exists(),
13344            "refused module executed store-touch sentinel"
13345        );
13346        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
13347    }
13348
13349    #[cfg(windows)]
13350    #[tokio::test]
13351    async fn windows_nested_daemon_fixture() {
13352        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
13353            return;
13354        };
13355        let mut spec = spec();
13356        spec.env.push((
13357            "SUBC_TEST_STORE_TOUCH".into(),
13358            sentinel.to_str().unwrap().into(),
13359        ));
13360        let supervisor =
13361            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13362        let module = supervisor
13363            .spawn(spec)
13364            .expect("per-module job must nest below daemon and host jobs");
13365        tokio::time::timeout(Duration::from_secs(10), async {
13366            while !PathBuf::from(&sentinel).exists() {
13367                tokio::task::yield_now().await;
13368            }
13369        })
13370        .await
13371        .unwrap();
13372        module.stop().await.unwrap();
13373    }
13374
13375    #[cfg(windows)]
13376    #[test]
13377    fn windows_module_job_nests_under_host_job() {
13378        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
13379        let sentinel = home.join("nested-store-touched");
13380        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13381        command
13382            .args([
13383                "--exact",
13384                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
13385                "--nocapture",
13386            ])
13387            .env("SUBC_TEST_NESTED_STORE", &sentinel);
13388        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13389        assert!(daemon.child.wait().unwrap().success());
13390        assert!(sentinel.exists(), "nested protocol:none module never ran");
13391    }
13392}
13393
13394#[cfg(target_os = "linux")]
13395pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
13396    use subc_cgroup::KillOutcome;
13397    match subc_cgroup::kill_module(placement, module_id) {
13398        KillOutcome::Killed => {}
13399        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
13400            debug!(
13401                module_id,
13402                "cgroup tree kill unavailable; using direct-child kill"
13403            );
13404        }
13405        KillOutcome::IoError { path, error } => {
13406            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
13407        }
13408    }
13409}
13410
13411/// Assign a freshly spawned Windows child without ever resuming on failure.
13412///
13413/// Steps 2 and 3 of the suspended-create contract: the job is created and the
13414/// child assigned **while it is still suspended** (step 1 is
13415/// `suspend_on_create_async` at the spawn site). The caller resumes only after
13416/// the child's roster admission is durable. Failed containment kills and reaps
13417/// the suspended child instead of running an unprotected module.
13418#[cfg(windows)]
13419fn contain_spawned_child(
13420    child: &mut Child,
13421    spec: &ModuleSpec,
13422    carried: Option<&Arc<containment::CarriedContainment>>,
13423) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
13424    let assignment = || {
13425        let job = match carried {
13426            Some(container) => container.job(),
13427            None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
13428        };
13429        #[cfg(test)]
13430        spawn_record_tests::assignment_failpoint(spec)?;
13431        job.lock()
13432            .unwrap_or_else(|p| p.into_inner())
13433            .assign(child)?;
13434        Ok::<_, io::Error>(job)
13435    };
13436    match assignment() {
13437        Ok(job) => Ok(Some(job)),
13438        Err(source) => {
13439            kill_and_reap_rejected_child(child);
13440            Err(SuperviseError::Spawn {
13441                program: spec.program.clone(),
13442                source: io::Error::other(format!(
13443                    "Windows module job containment failed: {source}"
13444                )),
13445                cgroup_path: None,
13446            })
13447        }
13448    }
13449}
13450
13451/// Rejection is synchronous: do not return a spawn error while the child can
13452/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
13453fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
13454    #[cfg(unix)]
13455    group.signal_rejected(child);
13456    let _ = child.start_kill();
13457    while matches!(child.try_wait(), Ok(None)) {
13458        std::thread::yield_now();
13459    }
13460}
13461
13462/// Resume a suspended child, killing it if it cannot be started.
13463///
13464/// A suspended process holds a pid and does nothing, so there is no useful
13465/// state to return: the caller gets an error and the spawn fails.
13466#[cfg(windows)]
13467fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
13468    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
13469        // Kill it here rather than leaving a suspended process for the caller
13470        // to notice; `kill_on_drop` would eventually do this, but the module
13471        // would have been reported as running in between.
13472        let _ = subc_os::child_spawn::status(
13473            std::process::Command::new("taskkill.exe")
13474                .args(["/PID", &pid.to_string(), "/T", "/F"])
13475                .stdin(Stdio::null())
13476                .stdout(Stdio::null())
13477                .stderr(Stdio::null()),
13478        );
13479        return Err(SuperviseError::Spawn {
13480            program: spec.program.clone(),
13481            source,
13482            cgroup_path: None,
13483        });
13484    }
13485    Ok(())
13486}
13487
13488#[cfg(target_os = "linux")]
13489fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13490    match placement.remove_module(module_id) {
13491        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
13492        Err(error) => warn!(
13493            module_id,
13494            error = %error,
13495            "could not remove module cgroup after process exit; continuing teardown"
13496        ),
13497    }
13498}
13499
13500#[cfg(target_os = "linux")]
13501const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
13502#[cfg(target_os = "linux")]
13503const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
13504
13505#[cfg(target_os = "linux")]
13506async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13507    let path = placement.module_path(module_id).ok();
13508    if path.as_ref().is_some_and(|path| {
13509        std::fs::read_to_string(path.join("cgroup.events"))
13510            .ok()
13511            .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
13512    }) {
13513        remove_module_cgroup(placement, module_id);
13514        return;
13515    }
13516
13517    // Freeze before enumerating: a frozen task cannot exit and release its PID
13518    // for reuse by a process outside this module while SIGTERM is being sent.
13519    if terminate_reaped_cgroup(placement, module_id).await {
13520        if let Some(path) = &path {
13521            let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
13522            while cgroup_is_populated(path) && Instant::now() < deadline {
13523                sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
13524            }
13525        }
13526    }
13527
13528    // Reaping the direct child is not proof its descendants exited. Kill any
13529    // survivors; when `cgroup.kill` succeeds, wait for the kernel to report the
13530    // cgroup empty before rmdir.
13531    if matches!(
13532        subc_cgroup::kill_module(Some(placement), module_id),
13533        subc_cgroup::KillOutcome::Killed
13534    ) {
13535        if let Some(path) = &path {
13536            while cgroup_is_populated(path) {
13537                sleep(Duration::from_millis(1)).await;
13538            }
13539        }
13540    }
13541    remove_module_cgroup(placement, module_id);
13542}
13543
13544#[cfg(target_os = "linux")]
13545fn cgroup_is_populated(path: &std::path::Path) -> bool {
13546    std::fs::read_to_string(path.join("cgroup.events"))
13547        .ok()
13548        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
13549}
13550
13551#[cfg(target_os = "linux")]
13552async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
13553    let frozen =
13554        match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
13555            .await
13556        {
13557            subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
13558            outcome => {
13559                tracing::debug!(
13560                    module_id,
13561                    ?outcome,
13562                    "cgroup freeze failed; skipping SIGTERM grace"
13563                );
13564                return false;
13565            }
13566        };
13567    let listed = match frozen.processes() {
13568        subc_cgroup::ProcsOutcome::Listed(pids) => {
13569            for pid in pids.into_iter().filter(|pid| *pid > 1) {
13570                if let Some(pid) = rustix::process::Pid::from_raw(pid) {
13571                    if let Err(error) =
13572                        rustix::process::kill_process(pid, rustix::process::Signal::TERM)
13573                    {
13574                        tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
13575                    }
13576                }
13577            }
13578            true
13579        }
13580        outcome => {
13581            tracing::debug!(
13582                module_id,
13583                ?outcome,
13584                "cgroup process listing failed; skipping SIGTERM grace"
13585            );
13586            false
13587        }
13588    };
13589    match frozen.thaw() {
13590        subc_cgroup::ThawOutcome::Thawed => listed,
13591        outcome => {
13592            tracing::debug!(
13593                module_id,
13594                ?outcome,
13595                "cgroup thaw failed; proceeding to cgroup.kill"
13596            );
13597            false
13598        }
13599    }
13600}
13601
13602#[cfg(target_os = "linux")]
13603fn apply_cgroup_placement(
13604    command: &mut Command,
13605    spec: &ModuleSpec,
13606    path: &std::path::Path,
13607) -> Result<(), SuperviseError> {
13608    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
13609        module_id: spec.module_id.clone(),
13610        source,
13611    })
13612}
13613
13614fn capture_retention(spec: &ModuleSpec) -> Retention {
13615    let defaults = Retention::default();
13616    let value = |name: &str| {
13617        spec.env
13618            .iter()
13619            .rev()
13620            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
13621    };
13622    Retention {
13623        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
13624            .and_then(|value| value.parse().ok())
13625            .unwrap_or(defaults.max_file_mb),
13626        keep: value(CAPTURE_KEEP_ENV)
13627            .and_then(|value| value.parse().ok())
13628            .unwrap_or(defaults.keep),
13629        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
13630            .and_then(|value| value.parse().ok())
13631            .unwrap_or(defaults.max_age_days),
13632    }
13633}
13634
13635/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
13636/// module's registration to the exact process the supervisor spawned.
13637fn generate_launch_nonce() -> Result<String, SuperviseError> {
13638    let mut bytes = [0u8; 32];
13639    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
13640        reason: source.to_string(),
13641    })?;
13642    let mut hex = String::with_capacity(64);
13643    for b in bytes {
13644        use std::fmt::Write;
13645        let _ = write!(hex, "{b:02x}");
13646    }
13647    Ok(hex)
13648}
13649
13650/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
13651/// signal about how many leading bytes matched.
13652fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
13653    if a.len() != b.len() {
13654        return false;
13655    }
13656    let mut diff = 0u8;
13657    for (x, y) in a.iter().zip(b.iter()) {
13658        diff |= x ^ y;
13659    }
13660    diff == 0
13661}
13662
13663/// The kernel's image after an acknowledged exec, shared by ordinary launches
13664/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
13665/// not identities inferred from a configured pathname.
13666#[cfg(not(windows))]
13667fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
13668    let process = subc_os::Process::open(pid).ok().flatten()?;
13669    #[cfg(target_os = "linux")]
13670    return sample_after_linux_exec(
13671        || {
13672            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
13673            let committed = linux_exec_committed(&stat);
13674            #[cfg(test)]
13675            spawn_image_tests::report_exec_state(committed);
13676            committed
13677        },
13678        || process.observe(),
13679    );
13680    #[cfg(not(target_os = "linux"))]
13681    process.observe()
13682}
13683
13684/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
13685/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
13686/// installation. Until it clears, /proc/pid/exe can still name the daemon.
13687#[cfg(target_os = "linux")]
13688fn linux_exec_committed(stat: &str) -> Option<bool> {
13689    const PF_FORKNOEXEC: u64 = 0x40;
13690    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
13691    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
13692}
13693
13694#[cfg(target_os = "linux")]
13695fn sample_after_linux_exec(
13696    mut exec_committed: impl FnMut() -> Option<bool>,
13697    observe: impl FnOnce() -> Option<subc_os::Observation>,
13698) -> Option<subc_os::Observation> {
13699    let deadline = std::time::Instant::now() + Duration::from_secs(1);
13700    while !exec_committed()? {
13701        // A failed or indefinitely stalled exec must not publish the inherited
13702        // image as authority to signal a future orphan. Missing identity keeps
13703        // the existing conservative orphan-sweep behavior.
13704        if std::time::Instant::now() >= deadline {
13705            return None;
13706        }
13707        std::thread::sleep(Duration::from_micros(50));
13708    }
13709    observe()
13710}
13711
13712#[cfg(all(test, target_os = "linux"))]
13713mod spawn_image_tests {
13714    use super::*;
13715    use std::cell::{Cell, RefCell};
13716
13717    enum ExecEvent {
13718        Uncommitted,
13719        Image(Option<subc_os::Observation>),
13720    }
13721
13722    thread_local! {
13723        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
13724    }
13725
13726    pub(super) fn report_exec_state(committed: Option<bool>) {
13727        if committed == Some(false) {
13728            EXEC_EVENTS.with(|events| {
13729                if let Some(events) = events.borrow_mut().take() {
13730                    let _ = events.send(ExecEvent::Uncommitted);
13731                }
13732            });
13733        }
13734    }
13735
13736    #[test]
13737    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
13738        use std::io::{Read, Write};
13739        use std::os::fd::AsRawFd;
13740
13741        let descriptor = std::fs::File::open("/dev/null").unwrap();
13742        let mut command = std::process::Command::new("/bin/sleep");
13743        command.arg("30");
13744        let (mut ready, mut release) =
13745            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
13746                .unwrap();
13747        let spawn = std::thread::spawn(move || command.spawn());
13748        let mut message = [0u8; 8];
13749        ready.read_exact(&mut message).unwrap();
13750        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
13751        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
13752        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
13753        let (events, received) = std::sync::mpsc::channel();
13754        let observer = std::thread::spawn(move || {
13755            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
13756            let image = observe_spawned_image(pid);
13757            let _ = events.send(ExecEvent::Image(image));
13758        });
13759        // Release only once the production observer has seen the uncommitted
13760        // exec, or has incorrectly returned the inherited image. Either way,
13761        // release and reap before asserting so a regression cannot leak a child.
13762        let first = received.recv_timeout(Duration::from_secs(5));
13763        release.write_all(&[1]).unwrap();
13764        let mut child = spawn.join().unwrap().unwrap();
13765        let after = match first {
13766            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
13767                Ok(ExecEvent::Image(image)) => image,
13768                _ => None,
13769            },
13770            Ok(ExecEvent::Image(image)) => image,
13771            Err(_) => None,
13772        };
13773        observer.join().unwrap();
13774        child.kill().unwrap();
13775        child.wait().unwrap();
13776
13777        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
13778        assert_eq!(
13779            inherited.unwrap().executable,
13780            subc_os::file_identity(&std::env::current_exe().unwrap())
13781        );
13782        assert_eq!(
13783            after.unwrap().executable,
13784            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
13785        );
13786    }
13787
13788    #[test]
13789    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
13790        let committed = Cell::new(false);
13791        let inherited = subc_os::Observation {
13792            start_time: 23,
13793            executable: Some(subc_os::FileIdentity {
13794                device: 29,
13795                inode: 31,
13796            }),
13797        };
13798        let module = subc_os::Observation {
13799            start_time: 23,
13800            executable: Some(subc_os::FileIdentity {
13801                device: 29,
13802                inode: 37,
13803            }),
13804        };
13805        let polls = Cell::new(0);
13806        let image = sample_after_linux_exec(
13807            || {
13808                polls.set(polls.get() + 1);
13809                // Hold the inherited image through the first poll, as in the
13810                // posix_spawn parent-wake window, then commit the module image.
13811                committed.set(polls.get() == 2);
13812                Some(committed.get())
13813            },
13814            || Some(if committed.get() { module } else { inherited }),
13815        );
13816        assert_eq!(image, Some(module));
13817        assert_eq!(polls.get(), 2);
13818    }
13819
13820    #[test]
13821    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
13822        assert_eq!(
13823            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
13824            Some(false)
13825        );
13826        assert_eq!(
13827            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
13828            Some(true)
13829        );
13830        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
13831        assert_eq!(linux_exec_committed("17 (a) R"), None);
13832    }
13833
13834    #[test]
13835    fn unreadable_exec_state_never_samples_an_inherited_image() {
13836        assert_eq!(
13837            sample_after_linux_exec(
13838                || None,
13839                || panic!("must not sample before exec confirmation")
13840            ),
13841            None
13842        );
13843    }
13844
13845    #[test]
13846    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
13847        assert_eq!(
13848            sample_after_linux_exec(
13849                || Some(false),
13850                || panic!("must not sample after an exec timeout")
13851            ),
13852            None
13853        );
13854    }
13855}
13856
13857/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
13858/// different executable after the no-delete capture was taken.
13859#[cfg(windows)]
13860pub(crate) fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
13861    if let Err(error) = require_program_search_path(spec) {
13862        return Err(match error {
13863            SuperviseError::Spawn { source, .. } => source,
13864            error => io::Error::other(error),
13865        });
13866    }
13867    let mut program = spec.program.clone();
13868    if program.is_absolute() || program.components().count() > 1 {
13869        // An explicit path can name a PE image with no extension. Try that
13870        // exact file first, as Command does, before falling back to .exe.
13871        match subc_os::ExecutableCapture::open(&program) {
13872            Ok(capture) => return Ok(capture),
13873            Err(error)
13874                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
13875            Err(error) => return Err(error),
13876        }
13877        program.set_extension("exe");
13878        return subc_os::ExecutableCapture::open(&program);
13879    }
13880    if program.extension().is_none() {
13881        program.set_extension("exe");
13882    }
13883    let custom_path = spec
13884        .env
13885        .iter()
13886        .rev()
13887        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
13888    let mut directories = Vec::new();
13889    // These are Windows' default executable search locations. A configured
13890    // PATH still overrides the inherited PATH, not the system directories.
13891    if let Some(parent) = std::env::current_exe()?.parent() {
13892        directories.push(parent.to_path_buf());
13893    }
13894    directories.push(std::env::current_dir()?);
13895    if let Some(windows) = std::env::var_os("SystemRoot") {
13896        let windows = PathBuf::from(windows);
13897        directories.push(windows.join("System32"));
13898        directories.push(windows.clone());
13899    }
13900    let path = custom_path
13901        .map(|(_, value)| std::ffi::OsString::from(value))
13902        .or_else(|| std::env::var_os("PATH"));
13903    if let Some(path) = path {
13904        directories.extend(std::env::split_paths(&path));
13905    }
13906    for directory in directories {
13907        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
13908            Ok(capture) => return Ok(capture),
13909            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
13910            Err(error) => return Err(error),
13911        }
13912    }
13913    Err(io::Error::new(
13914        io::ErrorKind::NotFound,
13915        "resolved executable not found",
13916    ))
13917}
13918
13919async fn spawn_and_mark_running_async(
13920    spec: &ModuleSpec,
13921    runtime: &SupervisorRuntimeConfig,
13922    snapshot: &SharedSnapshot,
13923) -> Result<SupervisedChild, SuperviseError> {
13924    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13925    let registration_watch = watch_registration(&runtime.registry, spec);
13926    #[cfg(unix)]
13927    let joining_group = containment::current(&runtime.child_roster.containment)
13928        .is_some_and(|container| container.group().is_some());
13929    let result = spawn_child_async(spec, runtime).await;
13930    #[cfg(unix)]
13931    let result = match result {
13932        Err(error)
13933            if joining_group
13934                && !is_launch_deferral(&error)
13935                && !runtime.child_roster.is_closed()
13936                && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13937                && runtime
13938                    .child_roster
13939                    .previous_instance(&spec.module_id)
13940                    .is_none() =>
13941        {
13942            // A failed group join has already ended the old container. Its single
13943            // fresh-container retry belongs to the same restart budget entry.
13944            warn!(module_id = %spec.module_id, %error, "carried process-group spawn failed; trying one fresh container");
13945            spawn_child_async(spec, runtime).await
13946        }
13947        other => other,
13948    };
13949    let mut child = result?;
13950    arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
13951    if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
13952        containment::terminate_blocking(&runtime.child_roster.containment);
13953        return Err(error);
13954    }
13955    if let Some(spawn) = activity_spawn {
13956        spawn.complete(snapshot);
13957    }
13958    Ok(child)
13959}
13960
13961#[cfg(test)]
13962fn spawn_and_mark_running(
13963    spec: &ModuleSpec,
13964    runtime: &SupervisorRuntimeConfig,
13965    snapshot: &SharedSnapshot,
13966) -> Result<SupervisedChild, SuperviseError> {
13967    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13968    let registration_watch = watch_registration(&runtime.registry, spec);
13969    #[cfg(unix)]
13970    let joining_group = containment::current(&runtime.child_roster.containment)
13971        .is_some_and(|container| container.group().is_some());
13972    let spawn = || {
13973        spawn_child(
13974            spec,
13975            runtime.connection_file_path.as_deref(),
13976            runtime.supervisor_handle.as_ref(),
13977            &runtime.stderr_ring,
13978            runtime.capture_logs_dir.as_deref(),
13979            &runtime.child_roster,
13980            runtime.spawn_delivery(),
13981            #[cfg(target_os = "linux")]
13982            runtime.cgroup_placement.as_ref(),
13983            runtime.register_timeout,
13984        )
13985    };
13986    let result = spawn();
13987    #[cfg(unix)]
13988    let result = match result {
13989        Err(error)
13990            if joining_group
13991                && !is_launch_deferral(&error)
13992                && !runtime.child_roster.is_closed()
13993                && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13994                && runtime
13995                    .child_roster
13996                    .previous_instance(&spec.module_id)
13997                    .is_none() =>
13998        {
13999            // The failed join already destroyed and verified the old container.
14000            // Try one fresh group as part of this same restart, without spending
14001            // another crash-budget entry. Cgroups and Windows jobs do not retry.
14002            warn!(module_id = %spec.module_id, %error, "carried process-group spawn failed; trying one fresh container");
14003            spawn()
14004        }
14005        other => other,
14006    };
14007    let child = result?;
14008    let mut child = child;
14009    arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
14010    if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
14011        containment::terminate_blocking(&runtime.child_roster.containment);
14012        return Err(error);
14013    }
14014    if let Some(spawn) = activity_spawn {
14015        spawn.complete(snapshot);
14016    }
14017    Ok(child)
14018}
14019
14020struct RegistrationWatch {
14021    changes: watch::Receiver<u64>,
14022    baseline: u64,
14023}
14024
14025// Capture before spawning: a fast child's accepted HELLO can disappear from the
14026// active registry before the monitor first runs. A pre-existing registration
14027// must not satisfy a new process's deadline either.
14028fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
14029    if spec.protocol != ModuleProtocol::Subc {
14030        return None;
14031    }
14032    let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
14033    let baseline = *changes.borrow();
14034    Some(RegistrationWatch { changes, baseline })
14035}
14036
14037fn arm_registration_deadline(
14038    child: &mut SupervisedChild,
14039    budget: Duration,
14040    registration_watch: Option<RegistrationWatch>,
14041) {
14042    child.registration_watch = registration_watch;
14043    child.registration_deadline = (child.protocol == ModuleProtocol::Subc
14044        && child.awaiting_registration())
14045    .then_some(child.spawned_at + budget);
14046}
14047
14048async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
14049    child.retire_registration();
14050    if let Some(container) = child.child.container() {
14051        container.stop_members().await;
14052        child.kill_leader()?;
14053        child.wait().await?;
14054        return Ok(());
14055    }
14056    #[cfg(unix)]
14057    {
14058        // Children lead their own process groups. Signal the whole group so a
14059        // helper stuck before HELLO cannot survive its supervised parent.
14060        let group = unreaped_registration_group(child)?;
14061        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
14062        // Process-group safety rule: only signal a group while its leader is
14063        // our unreaped child. Even a zombie leader reserves its pid/group id;
14064        // reaping it before KILL would let that id name an unrelated group.
14065        // Sleep without waiting so TERM-ignoring helpers get the full grace.
14066        sleep(crate::child_roster::TERM_TO_KILL).await;
14067        let group = unreaped_registration_group(child)?;
14068        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
14069    }
14070    // Windows terminates the containing Job Object as well as the direct
14071    // child. Linux also kills the child's cgroup when placement is enabled.
14072    child.start_kill()?;
14073    child.wait().await?;
14074    Ok(())
14075}
14076
14077#[cfg(unix)]
14078fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
14079    if let Some(container) = child.child.container() {
14080        return container
14081            .group()
14082            .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
14083            .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
14084    }
14085    // The group id comes only from the pid recorded at spawn, when the child
14086    // created its own group. Never admit init's group or a negative group id.
14087    let group = i32::try_from(child.pid)
14088        .ok()
14089        .filter(|pid| *pid > 1)
14090        .and_then(rustix::process::Pid::from_raw)
14091        .ok_or_else(|| {
14092            io::Error::new(
14093                io::ErrorKind::InvalidInput,
14094                "invalid supervised process group",
14095            )
14096        })?;
14097    if child.child.id() != Some(child.pid) {
14098        return Err(io::Error::new(
14099            io::ErrorKind::InvalidInput,
14100            "refusing process-group signal after supervised child was reaped",
14101        ));
14102    }
14103    Ok(group)
14104}
14105
14106enum RegistrationWaitOutcome {
14107    Registered,
14108    Exited(ExitReport),
14109    TimedOut,
14110}
14111
14112struct ReloadRegistrationFailure {
14113    exit_report: ExitReport,
14114    reason: String,
14115}
14116
14117#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14118enum BusyGaugeObservation {
14119    Quiescent,
14120    Busy,
14121    Omitted,
14122}
14123
14124/// What one health answer said about one declared busy gauge.
14125#[derive(Debug, Clone, PartialEq, Eq)]
14126enum GaugeVerdict {
14127    /// The gauge read 0: this gauge is not what holds the drain.
14128    Zero,
14129    /// The gauge read a positive count.
14130    NonZero(u64),
14131    /// The gauge was present but not a non-negative integer (`0.0`, a string,
14132    /// a negative number). It counts as busy, because the daemon cannot tell
14133    /// it is idle. Carries the value as JSON text, cut to a short prefix.
14134    NotACount(String),
14135    /// The health answer had no metrics object, or one without this gauge.
14136    Missing,
14137    /// The probe produced no health answer to read; carries the probe error's
14138    /// evidence label (`no-answer` for a probe that timed out).
14139    ProbeFailed(&'static str),
14140}
14141
14142impl GaugeVerdict {
14143    fn reads_quiet(&self) -> bool {
14144        matches!(self, Self::Zero)
14145    }
14146}
14147
14148impl fmt::Display for GaugeVerdict {
14149    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
14150        match self {
14151            Self::Zero => f.write_str("zero"),
14152            Self::NonZero(value) => write!(f, "non_zero({value})"),
14153            Self::NotACount(value) => write!(f, "not_a_count({value})"),
14154            Self::Missing => f.write_str("missing_from_health_body"),
14155            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
14156        }
14157    }
14158}
14159
14160/// The longest JSON text kept for a gauge value that is not a count, so a
14161/// module reporting a large object as a gauge cannot flood the drain log.
14162const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
14163
14164/// Read each declared gauge out of one health answer's metrics.
14165fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
14166    let metrics = metrics.and_then(Value::as_object);
14167    gauges
14168        .iter()
14169        .map(
14170            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
14171                None => GaugeVerdict::Missing,
14172                Some(value) => match value.as_u64() {
14173                    Some(0) => GaugeVerdict::Zero,
14174                    Some(count) => GaugeVerdict::NonZero(count),
14175                    None => {
14176                        let mut text = value.to_string();
14177                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
14178                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
14179                            while !text.is_char_boundary(end) {
14180                                end -= 1;
14181                            }
14182                            text.truncate(end);
14183                            text.push_str("...");
14184                        }
14185                        GaugeVerdict::NotACount(text)
14186                    }
14187                },
14188            },
14189        )
14190        .collect()
14191}
14192
14193/// The drain's overall reading of one health answer: omitted when any
14194/// declared gauge is missing, quiescent only when every gauge reads 0.
14195fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
14196    if verdicts
14197        .iter()
14198        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
14199    {
14200        BusyGaugeObservation::Omitted
14201    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
14202        BusyGaugeObservation::Quiescent
14203    } else {
14204        BusyGaugeObservation::Busy
14205    }
14206}
14207
14208/// One declared gauge's history over a drain: the latest verdict on it, and
14209/// the latest value actually read for it (which a later failed probe or
14210/// missing gauge does not erase). Times are offsets from the start of the
14211/// drain's wait.
14212#[derive(Debug, Clone, Default, PartialEq, Eq)]
14213struct GaugeTrace {
14214    latest: Option<(GaugeVerdict, Duration)>,
14215    last_value: Option<(GaugeVerdict, Duration)>,
14216}
14217
14218/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
14219/// its ceiling can say in its timeout line which declared gauge held it and
14220/// why. Without it, a gauge that read busy, a gauge the module never
14221/// reported, and a probe that never got an answer all looked the same: a
14222/// drain that ended at its ceiling with nothing in flight.
14223#[derive(Debug, Clone, Default, PartialEq, Eq)]
14224struct DrainGaugeTrace {
14225    /// Health probes the drain sent.
14226    probes: u32,
14227    /// Each declared gauge, in declaration order.
14228    gauges: Vec<(String, GaugeTrace)>,
14229}
14230
14231impl DrainGaugeTrace {
14232    fn new(gauges: &[String]) -> Self {
14233        Self {
14234            probes: 0,
14235            gauges: gauges
14236                .iter()
14237                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
14238                .collect(),
14239        }
14240    }
14241
14242    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
14243    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
14244        self.probes = self.probes.saturating_add(1);
14245        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
14246            trace.latest = Some((verdict.clone(), at));
14247            if matches!(
14248                verdict,
14249                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
14250            ) {
14251                trace.last_value = Some((verdict.clone(), at));
14252            }
14253        }
14254    }
14255}
14256
14257/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
14258/// declared gauge, separated by ` | `. A value is shown as its verdict
14259/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
14260/// and whether that counted as busy.
14261impl fmt::Display for DrainGaugeTrace {
14262    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
14263        if self.gauges.is_empty() {
14264            return f.write_str("none declared");
14265        }
14266        for (index, (name, trace)) in self.gauges.iter().enumerate() {
14267            if index > 0 {
14268                f.write_str(" | ")?;
14269            }
14270            write!(f, "{name}: ")?;
14271            match &trace.latest {
14272                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
14273                None => f.write_str("never probed")?,
14274            }
14275            match &trace.last_value {
14276                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
14277                None => f.write_str("; no value ever read")?,
14278            }
14279        }
14280        Ok(())
14281    }
14282}
14283
14284pub(crate) fn declared_busy_gauges(
14285    registry: &Registry,
14286    module_id: &str,
14287) -> Result<Vec<String>, SuperviseError> {
14288    busy_gauges_of(
14289        registry
14290            .get_module(module_id)
14291            .map_err(SuperviseError::Registry)?,
14292    )
14293}
14294
14295/// [`declared_busy_gauges`] for the registration a connection holds, in any
14296/// slot: after cutover the incumbent is no longer the id's active
14297/// registration, and its own manifest is the one that names its gauges.
14298fn declared_busy_gauges_for_connection(
14299    registry: &Registry,
14300    connection_id: ConnectionId,
14301) -> Result<Vec<String>, SuperviseError> {
14302    busy_gauges_of(
14303        registry
14304            .get_module_by_connection(connection_id)
14305            .map_err(SuperviseError::Registry)?,
14306    )
14307}
14308
14309fn busy_gauges_of(
14310    registration: Option<crate::registry::ModuleRegistration>,
14311) -> Result<Vec<String>, SuperviseError> {
14312    let Some(registration) = registration else {
14313        return Ok(Vec::new());
14314    };
14315    let Some(self_signals) = registration.manifest.self_signals else {
14316        return Ok(Vec::new());
14317    };
14318
14319    let mut gauges = Vec::new();
14320    for declaration in self_signals {
14321        if declaration.kind != SelfSignalKind::Busy {
14322            continue;
14323        }
14324        match declaration.anchored_to {
14325            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
14326                gauges.extend(declared)
14327            }
14328            _ => {
14329                // An invalid Busy anchor is fail-safe: the empty name cannot be
14330                // present in a conforming health report, so this drain stays busy.
14331                gauges.push(String::new());
14332            }
14333        }
14334    }
14335    Ok(gauges)
14336}
14337
14338/// Wait for `endpoint` to have nothing in flight and, when the module declares
14339/// busy gauges, for a health probe to report them quiet. The probe is addressed
14340/// by `scope`: a swap's superseded incumbent must be asked about its own
14341/// gauges, and by module id the probe would reach the promoted candidate.
14342/// While the gauges read busy, the probe repeats every
14343/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
14344/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
14345/// timeout line.
14346#[allow(clippy::too_many_arguments)]
14347async fn wait_for_forwarding_quiescence(
14348    forwarding: &ForwardingTable,
14349    module_id: &str,
14350    runtime: &SupervisorRuntimeConfig,
14351    endpoint: crate::ModuleEndpointId,
14352    deadline: Instant,
14353    busy_gauges: &[String],
14354    scope: DrainScope,
14355    trace: &mut DrainGaugeTrace,
14356) -> Result<bool, SuperviseError> {
14357    let wait_started = Instant::now();
14358    let mut gauges_quiescent = busy_gauges.is_empty();
14359    let mut next_probe_at = wait_started;
14360    let mut omission_reported = false;
14361
14362    loop {
14363        let now = Instant::now();
14364        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
14365            let report = match scope {
14366                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
14367                DrainScope::Endpoint(endpoint) => {
14368                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
14369                }
14370            };
14371            let verdicts = match &report {
14372                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
14373                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
14374            };
14375            trace.record(&verdicts, wait_started.elapsed());
14376            gauges_quiescent = match report {
14377                Ok(_) => match busy_gauge_observation(&verdicts) {
14378                    BusyGaugeObservation::Quiescent => true,
14379                    BusyGaugeObservation::Busy => false,
14380                    BusyGaugeObservation::Omitted => {
14381                        // Once per drain: a module that never reports a gauge
14382                        // it declared would otherwise say so on every probe.
14383                        if !omission_reported {
14384                            forwarding
14385                                .counters()
14386                                .increment_drains_with_undeclared_gauge();
14387                            let missing = busy_gauges
14388                                .iter()
14389                                .zip(&verdicts)
14390                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
14391                                .map(|(gauge, _)| gauge.as_str())
14392                                .collect::<Vec<_>>()
14393                                .join(",");
14394                            warn!(
14395                                module_id,
14396                                missing_gauges = %missing,
14397                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
14398                            );
14399                            omission_reported = true;
14400                        }
14401                        false
14402                    }
14403                },
14404                Err(err) => {
14405                    warn!(
14406                        module_id,
14407                        error = %err,
14408                        "drain health.check did not produce declared busy gauges; treating module as busy"
14409                    );
14410                    false
14411                }
14412            };
14413            // Counted from the probe's completion: the probe above was awaited,
14414            // so no second probe can start while one is still outstanding.
14415            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
14416        }
14417
14418        let in_flight = forwarding
14419            .endpoint_in_flight_count(endpoint)
14420            .map_err(SuperviseError::Forwarding)?;
14421        if in_flight == 0 && gauges_quiescent {
14422            return Ok(true);
14423        }
14424
14425        let now = Instant::now();
14426        if now >= deadline {
14427            return Ok(false);
14428        }
14429        let mut wait = deadline
14430            .saturating_duration_since(now)
14431            .min(REGISTRY_RELEASE_POLL);
14432        if !busy_gauges.is_empty() {
14433            wait = wait.min(next_probe_at.saturating_duration_since(now));
14434        }
14435        sleep(wait).await;
14436    }
14437}
14438
14439/// The wait between drain busy-gauge probes: the larger of
14440/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
14441/// drain's own in-flight poll interval.
14442fn drain_busy_reprobe_interval() -> Duration {
14443    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
14444}
14445
14446/// Logs why a drain gave up: the drain waited its full timeout without the
14447/// module going quiet, and teardown is being forced.
14448///
14449/// The line names two kinds of evidence, because a bare "timed out" hides
14450/// causes that need different fixes:
14451/// - `holdouts`: the requests and routes still open when the timeout hit,
14452///   and on which connections. One stuck call is a module bug; a held stream
14453///   on every session is a module that should end its streams when it
14454///   receives `module.draining`.
14455/// - `gauges`: for each busy gauge the module declares, its latest reading
14456///   and how many probes ran. This separates a module that kept reporting
14457///   itself busy, one whose health answer never contained the gauge, and one
14458///   that never answered the probes, which otherwise all look like a timeout
14459///   with nothing in flight.
14460fn warn_drain_timed_out(
14461    module_id: &str,
14462    drain_timeout: Duration,
14463    reason: RouteCloseReason,
14464    holdouts: &crate::forwarding::DrainHoldouts,
14465    gauges: &DrainGaugeTrace,
14466) {
14467    warn!(
14468        module_id = %module_id,
14469        waited = ?drain_timeout,
14470        ?reason,
14471        held_requests = holdouts.requests,
14472        held_routes = holdouts.routes,
14473        total_routes = holdouts.total_routes,
14474        top_connections = ?holdouts.top_connections,
14475        // `module_channel:corr`, so the module can find each held request
14476        // in its own log; capped, so `held_requests` is the full count.
14477        held = %holdouts
14478            .held
14479            .iter()
14480            .map(|(channel, corr)| format!("{channel}:{corr}"))
14481            .collect::<Vec<_>>()
14482            .join(","),
14483        gauge_probes = gauges.probes,
14484        busy_gauges = %gauges,
14485        "route drain timed out before request quiescence; forcing teardown"
14486    );
14487}
14488
14489/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
14490///
14491/// `Ok` is always honest and passed straight through -- the wait actually measured
14492/// in-flight state. `Err` means the wait produced no measurement at all (the
14493/// forwarding table's lock was poisoned), so `false` is reported as the one honest
14494/// constant: the drain did not complete. Never recomputed from route state, never a
14495/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
14496fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
14497    match wait_result {
14498        Ok(drained) => *drained,
14499        Err(_) => false,
14500    }
14501}
14502
14503fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
14504    for released in released_routes {
14505        let frame = match Frame::build_with_version(
14506            released.negotiated_ver,
14507            FrameType::Goodbye,
14508            control_flags(),
14509            released.channel,
14510            released.epoch,
14511            0,
14512            Vec::new(),
14513        ) {
14514            Ok(frame) => frame,
14515            Err(err) => {
14516                warn!(
14517                    route_channel = released.channel,
14518                    error = %err,
14519                    "failed to build supervisor drain route GOODBYE frame"
14520                );
14521                continue;
14522            }
14523        };
14524        if !released.close_on_delivery_failure() {
14525            crate::forwarding::send_module_route_goodbye(
14526                &forwarding.counters(),
14527                &released.sink,
14528                frame,
14529                released.module_id.as_deref(),
14530                "supervisor drain",
14531            );
14532            continue;
14533        }
14534        if let Err(err) = released.sink.try_send(frame) {
14535            warn!(
14536                target_connection_id = released.connection_id.get(),
14537                route_channel = released.channel,
14538                error = %err,
14539                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
14540            );
14541            let _ = forwarding.escalate_client_delivery_failure(
14542                released.connection_id,
14543                released.channel,
14544                released.epoch,
14545                CloseReason::new(
14546                    "route_goodbye_delivery_failed",
14547                    format!(
14548                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
14549                        released.channel
14550                    ),
14551                ),
14552                crate::forwarding::UndeliveredFrame {
14553                    module_id: released.module_id.as_deref(),
14554                    sink: &released.sink,
14555                },
14556            );
14557        }
14558    }
14559}
14560
14561async fn send_module_draining(
14562    module_id: &str,
14563    reason: RouteCloseReason,
14564    deadline_ms: u64,
14565    target: &ModuleDrainTarget,
14566    _wait_for_flush: bool,
14567) -> bool {
14568    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
14569        reason,
14570        deadline_ms,
14571    }) {
14572        Ok(body) => body,
14573        Err(err) => {
14574            warn!(
14575                module_id,
14576                error = %err,
14577                "failed to encode module draining command"
14578            );
14579            return false;
14580        }
14581    };
14582    let frame = match Frame::build_with_version(
14583        target.negotiated_ver,
14584        FrameType::Push,
14585        control_flags(),
14586        0,
14587        0,
14588        0,
14589        body,
14590    ) {
14591        Ok(frame) => frame,
14592        Err(err) => {
14593            warn!(
14594                module_id,
14595                error = %err,
14596                "failed to build module draining command frame"
14597            );
14598            return false;
14599        }
14600    };
14601    #[cfg(unix)]
14602    if _wait_for_flush {
14603        return target.sink.send_flushed(frame).await.is_ok();
14604    }
14605    if let Err(err) = target.sink.try_send(frame) {
14606        warn!(
14607            module_id,
14608            target_connection_id = target.endpoint.connection_id.get(),
14609            error = %err,
14610            "module draining command was not delivered to peer"
14611        );
14612        return false;
14613    }
14614    true
14615}
14616
14617/// The channel-0 GOODBYE that tells a module its stop is planned.
14618fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
14619    match Frame::build_with_version(
14620        negotiated_ver,
14621        FrameType::Goodbye,
14622        control_flags(),
14623        0,
14624        0,
14625        0,
14626        Vec::new(),
14627    ) {
14628        Ok(frame) => Some(frame),
14629        Err(err) => {
14630            warn!(
14631                module_id,
14632                error = %err,
14633                "failed to build module GOODBYE frame"
14634            );
14635            None
14636        }
14637    }
14638}
14639
14640/// Send every registered module connection its module GOODBYE at daemon
14641/// shutdown, then request that connection's close.
14642///
14643/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
14644/// before EOF, so the GOODBYE must reach the socket before the close. A close
14645/// request does not wait for the connection's queued frames: its writer gets a
14646/// bounded grace after the close, is aborted if it overruns it, and the daemon
14647/// process may exit before that grace ends. So with `wait_for_flush`, each
14648/// connection is closed only after its writer has acknowledged writing the
14649/// GOODBYE, or once a short shared budget runs out, so one module that is not
14650/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
14651/// are only queued, for a shutdown the operator has told to stop waiting.
14652/// A connection that is already gone is skipped.
14653#[cfg(unix)]
14654async fn send_module_goodbyes_for_daemon_shutdown(
14655    forwarding: &Arc<ForwardingTable>,
14656    reason: &CloseReason,
14657    wait_for_flush: bool,
14658) {
14659    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
14660    let targets = match forwarding.module_connections() {
14661        Ok(targets) => targets,
14662        Err(err) => {
14663            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
14664            return;
14665        }
14666    };
14667    let deadline = Instant::now() + GOODBYE_BUDGET;
14668    let mut sends = tokio::task::JoinSet::new();
14669    for target in targets {
14670        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
14671            continue;
14672        };
14673        if !wait_for_flush {
14674            if let Err(err) = target.sink.try_send(frame) {
14675                debug!(
14676                    module_id = %target.module_id,
14677                    error = %err,
14678                    "shutdown module GOODBYE was not queued"
14679                );
14680            }
14681            continue;
14682        }
14683        let forwarding = Arc::clone(forwarding);
14684        let reason = reason.clone();
14685        sends.spawn(async move {
14686            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
14687                Ok(Ok(())) => {}
14688                Ok(Err(err)) => debug!(
14689                    module_id = %target.module_id,
14690                    error = %err,
14691                    "module connection closed before its shutdown GOODBYE was written"
14692                ),
14693                Err(_) => warn!(
14694                    module_id = %target.module_id,
14695                    budget = ?GOODBYE_BUDGET,
14696                    "shutdown module GOODBYE was not written within its budget; closing anyway"
14697                ),
14698            }
14699            forwarding.request_connection_close(target.endpoint.connection_id, reason);
14700        });
14701    }
14702    // Every task ends by the shared deadline, so this wait is bounded too.
14703    while sends.join_next().await.is_some() {}
14704}
14705
14706async fn send_module_goodbye(
14707    module_id: &str,
14708    forwarding: &ForwardingTable,
14709    target: &ModuleDrainTarget,
14710    wait_for_flush: bool,
14711) {
14712    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
14713        return;
14714    };
14715    // A channel is shut down synchronously after its drain. Its final frame
14716    // must reach the socket writer first; queue admission alone can lose it.
14717    let sent = if wait_for_flush {
14718        timeout(Duration::from_millis(500), target.sink.send_flushed(frame))
14719            .await
14720            .map_err(|_| "module GOODBYE flush exceeded 500ms".to_owned())
14721            .and_then(|result| result.map_err(|err| err.to_string()))
14722    } else {
14723        target.sink.try_send(frame).map_err(|err| err.to_string())
14724    };
14725    if let Err(err) = sent {
14726        warn!(
14727            module_id,
14728            target_connection_id = target.endpoint.connection_id.get(),
14729            error = %err,
14730            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
14731        );
14732        forwarding.request_connection_close(
14733            target.endpoint.connection_id,
14734            CloseReason::new(
14735                "module_goodbye_delivery_failed",
14736                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
14737            ),
14738        );
14739    }
14740}
14741
14742#[derive(Clone, Copy)]
14743struct ForwardingDrainContext<'a> {
14744    spec: &'a ModuleSpec,
14745    runtime: &'a SupervisorRuntimeConfig,
14746    registry: &'a Registry,
14747    scope: DrainScope,
14748}
14749
14750/// Which process a forwarding drain addresses.
14751#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14752enum DrainScope {
14753    /// Whatever endpoint is active for the module id: every plain stop,
14754    /// restart and reload. Also moves the module's state to `Draining`.
14755    Active,
14756    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
14757    /// module id would resolve to the promoted candidate and leave neither
14758    /// process routable. The module's state is left alone, since the promoted
14759    /// candidate is what it describes and that process is running.
14760    Endpoint(crate::ModuleEndpointId),
14761}
14762
14763/// Whether a child being drained has already been asked to stop by the time
14764/// its drain wait starts.
14765///
14766/// The drain wait is the same budget whatever this says. What it decides is
14767/// whether the supervisor must ask by signal before that wait begins: a child
14768/// that nobody asked will sit out the whole budget and then be SIGKILLed,
14769/// healthy or not.
14770#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14771enum StopNotice {
14772    /// The module was sent `module.draining` and a module GOODBYE over its own
14773    /// registered connection, and stops itself.
14774    SentOverConnection,
14775    /// The forwarding drain found no registered connection for the module: a
14776    /// subc child spawned moments ago that has not sent HELLO yet, or a
14777    /// `protocol: "none"` child, which never registers.
14778    NoConnection,
14779    /// This path sends nothing over the module's connection: the supervisor has
14780    /// no forwarding table, or the caller stops the child without a forwarding
14781    /// drain.
14782    NotSent,
14783}
14784
14785async fn begin_forwarding_drain(
14786    spec: &ModuleSpec,
14787    runtime: &SupervisorRuntimeConfig,
14788    registry: &Registry,
14789    snapshot: &SharedSnapshot,
14790    enabled: Option<bool>,
14791    reason: RouteCloseReason,
14792) -> Result<StopNotice, SuperviseError> {
14793    let Some(forwarding) = runtime.forwarding.as_ref() else {
14794        return Err(SuperviseError::ReloadUnavailable {
14795            module_id: spec.module_id.clone(),
14796            reason: "supervisor was not configured with a forwarding table".to_string(),
14797        });
14798    };
14799
14800    begin_forwarding_drain_with(
14801        forwarding,
14802        ForwardingDrainContext {
14803            spec,
14804            runtime,
14805            registry,
14806            scope: DrainScope::Active,
14807        },
14808        snapshot,
14809        enabled,
14810        reason,
14811        runtime.drain_timeout,
14812    )
14813    .await
14814}
14815
14816async fn begin_forwarding_drain_if_configured(
14817    spec: &ModuleSpec,
14818    runtime: &SupervisorRuntimeConfig,
14819    registry: &Registry,
14820    snapshot: &SharedSnapshot,
14821    enabled: Option<bool>,
14822    reason: RouteCloseReason,
14823) -> Result<StopNotice, SuperviseError> {
14824    begin_forwarding_drain_with_timeout(
14825        spec,
14826        runtime,
14827        registry,
14828        snapshot,
14829        enabled,
14830        reason,
14831        runtime.drain_timeout,
14832    )
14833    .await
14834}
14835
14836/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
14837/// budget, for paths where the operator overrides the module's configured one
14838/// (`supervisor.restart{drain_timeout_ms}`).
14839async fn begin_forwarding_drain_with_timeout(
14840    spec: &ModuleSpec,
14841    runtime: &SupervisorRuntimeConfig,
14842    registry: &Registry,
14843    snapshot: &SharedSnapshot,
14844    enabled: Option<bool>,
14845    reason: RouteCloseReason,
14846    drain_timeout: Duration,
14847) -> Result<StopNotice, SuperviseError> {
14848    let Some(forwarding) = runtime.forwarding.as_ref() else {
14849        return Ok(StopNotice::NotSent);
14850    };
14851
14852    begin_forwarding_drain_with(
14853        forwarding,
14854        ForwardingDrainContext {
14855            spec,
14856            runtime,
14857            registry,
14858            scope: DrainScope::Active,
14859        },
14860        snapshot,
14861        enabled,
14862        reason,
14863        drain_timeout,
14864    )
14865    .await
14866}
14867
14868async fn begin_forwarding_drain_with(
14869    forwarding: &ForwardingTable,
14870    context: ForwardingDrainContext<'_>,
14871    snapshot: &SharedSnapshot,
14872    enabled: Option<bool>,
14873    reason: RouteCloseReason,
14874    drain_timeout: Duration,
14875) -> Result<StopNotice, SuperviseError> {
14876    let ForwardingDrainContext {
14877        spec,
14878        runtime,
14879        registry,
14880        scope,
14881    } = context;
14882    debug_assert_ne!(reason, RouteCloseReason::Crash);
14883    let terminal = matches!(reason, RouteCloseReason::Disable);
14884    let drain_started_at = Instant::now();
14885    let drain_deadline = drain_started_at + drain_timeout;
14886    let deadline_ms =
14887        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
14888    let busy_gauges = match scope {
14889        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
14890        DrainScope::Endpoint(endpoint) => {
14891            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
14892        }
14893    };
14894
14895    // Admission gate first: route.open/commit and route REQUEST admission are closed
14896    // before the first quiescence check, so the outstanding count can only fall.
14897    let gate_started = Instant::now();
14898    let drain_target = match scope {
14899        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
14900        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
14901    }
14902    .map_err(SuperviseError::Forwarding)?;
14903    // The instant admission closed, and how long taking the forwarding write
14904    // lock to close it took. The timeout line reports only the quiescence
14905    // wait, so without this a drain that started late looked like one that
14906    // started on time.
14907    info!(
14908        module_id = %spec.module_id,
14909        ?reason,
14910        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
14911        connected = drain_target.is_some(),
14912        "module drain began; route admission closed"
14913    );
14914    if scope == DrainScope::Active {
14915        update_snapshot(snapshot, Some(&spec.module_id), |state| {
14916            state.state = ModuleState::Draining;
14917            state.draining_to_replace =
14918                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
14919            if let Some(enabled) = enabled {
14920                state.enabled = enabled;
14921            }
14922        })?;
14923    }
14924
14925    let Some(target) = drain_target.as_ref() else {
14926        // Nothing was sent: the module has no registered connection to carry
14927        // `module.draining` or a GOODBYE. The caller must not assume the child
14928        // was asked to stop.
14929        return Ok(StopNotice::NoConnection);
14930    };
14931    let flush_goodbye = runtime.supervisor_handle.as_ref().is_some_and(|handle| {
14932        handle
14933            .registration_spawn(target.endpoint.connection_id)
14934            .is_some()
14935    });
14936    {
14937        send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
14938        #[cfg(feature = "test-support")]
14939        forwarding.wait_before_route_closing_for_test().await;
14940        let routes = forwarding
14941            .endpoint_routes(target.endpoint)
14942            .map_err(SuperviseError::Forwarding)?;
14943        let routes_notified = routes.len();
14944        crate::control::send_route_control_pushes(
14945            forwarding,
14946            routes.clone(),
14947            ClientControlPush::RouteClosing {
14948                module_id: spec.module_id.clone(),
14949                channels: Vec::new(),
14950                reason,
14951            },
14952        );
14953        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
14954
14955        // `route.closing` was just sent above: from here on every return path,
14956        // including an early one, MUST send `route.closed` before propagating
14957        // anything else. A client holds `closing` as a promise that a verdict is
14958        // coming; leaving early without `closed` strands it waiting forever, since
14959        // `closing` carries no timeout of its own.
14960        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
14961        let wait_result = wait_for_forwarding_quiescence(
14962            forwarding,
14963            &spec.module_id,
14964            runtime,
14965            target.endpoint,
14966            drain_deadline,
14967            &busy_gauges,
14968            scope,
14969            &mut gauge_trace,
14970        )
14971        .await;
14972        let drained = drained_after_quiescence_wait(&wait_result);
14973        if let Err(err) = &wait_result {
14974            error!(
14975                module_id = %spec.module_id,
14976                ?reason,
14977                error = %err,
14978                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
14979            );
14980        } else if !drained {
14981            // Read before teardown releases the routes.
14982            let holdouts = forwarding
14983                .endpoint_drain_holdouts(target.endpoint)
14984                .unwrap_or_default();
14985            warn_drain_timed_out(
14986                &spec.module_id,
14987                drain_timeout,
14988                reason,
14989                &holdouts,
14990                &gauge_trace,
14991            );
14992        }
14993        crate::control::send_route_control_pushes(
14994            forwarding,
14995            routes,
14996            ClientControlPush::RouteClosed {
14997                module_id: spec.module_id.clone(),
14998                channels: Vec::new(),
14999                reason,
15000                drained,
15001                abandoned: target.abandoned_bindings.len() as u32,
15002                excluded_subscriptions: target.excluded_subscriptions,
15003                terminal: Some(terminal),
15004            },
15005        );
15006        wait_result?;
15007
15008        // `route.closed` has now been sent unconditionally above. From here the
15009        // remaining steps are cleanup (route + module GOODBYE) rather than a
15010        // promise the client is waiting on, but a lock-poisoned
15011        // `release_module_endpoint_routes` would otherwise skip the module
15012        // GOODBYE silently too -- send it before propagating the error.
15013        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
15014            Ok(routes) => routes,
15015            Err(err) => {
15016                warn!(
15017                    module_id = %spec.module_id,
15018                    ?reason,
15019                    error = %err,
15020                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
15021                );
15022                send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
15023                return Err(SuperviseError::Forwarding(err));
15024            }
15025        };
15026        let route_goodbye_count = released_routes.len();
15027        send_route_goodbyes(forwarding, released_routes);
15028        send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
15029
15030        // The drain's happy path was previously silent: every emission above is
15031        // best-effort with only its failure arm logged, so "were consumers told"
15032        // was unprovable from the daemon log (surfaced by a 30-minute consumer
15033        // hang where the open question was exactly whether teardown notice went
15034        // out). One summary line makes that class decidable in one grep.
15035        info!(
15036            module_id = %spec.module_id,
15037            ?reason,
15038            routes_notified,
15039            route_goodbyes = route_goodbye_count,
15040            abandoned_reservations = target.abandoned_bindings.len(),
15041            excluded_subscriptions = target.excluded_subscriptions,
15042            drained,
15043            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
15044        );
15045    }
15046
15047    Ok(StopNotice::SentOverConnection)
15048}
15049
15050/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
15051/// the only slot a plain (non-swap) spawn can register into.
15052async fn wait_for_registration_after_reload(
15053    registry: &Registry,
15054    module_id: &str,
15055    snapshot: &SharedSnapshot,
15056    child: &mut SupervisedChild,
15057    wait: Duration,
15058) -> Result<RegistrationWaitOutcome, SuperviseError> {
15059    wait_for_slot_registration(
15060        registry,
15061        crate::registry::RegistrationSlot::Active(module_id),
15062        module_id,
15063        snapshot,
15064        child,
15065        wait,
15066    )
15067    .await
15068}
15069
15070/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
15071///
15072/// Keyed on the slot rather than the bare module id because during a swap the
15073/// id's active slot is already held by the incumbent: an id-keyed wait would
15074/// report the incumbent's registration as the candidate's and a candidate that
15075/// never registers would look registered. A swap candidate waits on
15076/// `crate::registry::RegistrationSlot::Candidate`.
15077async fn wait_for_slot_registration(
15078    registry: &Registry,
15079    slot: crate::registry::RegistrationSlot<'_>,
15080    module_id: &str,
15081    snapshot: &SharedSnapshot,
15082    child: &mut SupervisedChild,
15083    wait: Duration,
15084) -> Result<RegistrationWaitOutcome, SuperviseError> {
15085    let deadline = Instant::now() + wait;
15086    loop {
15087        if registry
15088            .registration(slot)
15089            .map_err(SuperviseError::Registry)?
15090            .is_some()
15091        {
15092            return Ok(RegistrationWaitOutcome::Registered);
15093        }
15094
15095        let now = Instant::now();
15096        if now >= deadline {
15097            return Ok(RegistrationWaitOutcome::TimedOut);
15098        }
15099        let remaining = deadline.saturating_duration_since(now);
15100        let poll = remaining.min(REGISTRY_RELEASE_POLL);
15101
15102        tokio::select! {
15103            wait_result = child.wait() => {
15104                let status = wait_result.map_err(|source| SuperviseError::Wait {
15105                    module_id: module_id.to_string(),
15106                    source,
15107                })?;
15108                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
15109                    snapshot,
15110                    child,
15111                    &status,
15112                )));
15113            }
15114            _ = sleep(poll) => {}
15115        }
15116    }
15117}
15118
15119fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
15120    // A process stopped for missing its first HELLO did not provide service,
15121    // even if its TERM handler exits zero. Reload also uses this classification
15122    // when a replacement exits before registering.
15123    if exit_report.kind != ExitKind::DeliberateSeverance {
15124        exit_report.kind = ExitKind::Crash;
15125    }
15126    exit_report
15127}
15128
15129async fn handle_reload_child_registration_failure(
15130    spec: &ModuleSpec,
15131    runtime: &SupervisorRuntimeConfig,
15132    registry: &Registry,
15133    process_liveness: &SupervisorProcessLiveness,
15134    snapshot: &SharedSnapshot,
15135    _child: &mut Option<SupervisedChild>,
15136    failure: ReloadRegistrationFailure,
15137) -> Result<(), SuperviseError> {
15138    let ReloadRegistrationFailure {
15139        exit_report,
15140        reason,
15141    } = failure;
15142    match on_child_exit(
15143        spec,
15144        runtime.restart_policy,
15145        registry,
15146        snapshot,
15147        &runtime.terminal_ring,
15148        &runtime.spawn_events,
15149        &runtime.child_roster,
15150        exit_report,
15151        runtime.recovery_is_daemon,
15152    )
15153    .await
15154    {
15155        NextAction::Stop {
15156            registration_released,
15157        } => {
15158            if registration_released {
15159                process_liveness.untrack_if_current(&spec.module_id, snapshot);
15160            }
15161        }
15162        NextAction::Restart { schedule } => {
15163            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
15164                schedule.delay
15165            });
15166            if let Some(schedule) = schedule {
15167                log_crash_respawn(&spec.module_id, schedule);
15168            }
15169            schedule_respawn(
15170                runtime,
15171                snapshot,
15172                &spec.module_id,
15173                delay,
15174                RespawnKind::Spawn,
15175                true,
15176            )?;
15177        }
15178    }
15179    Err(SuperviseError::ReloadFailed {
15180        module_id: spec.module_id.clone(),
15181        reason,
15182    })
15183}
15184
15185async fn handle_reload_spawn_failure(
15186    spec: &ModuleSpec,
15187    runtime: &SupervisorRuntimeConfig,
15188    process_liveness: &SupervisorProcessLiveness,
15189    snapshot: &SharedSnapshot,
15190    _child: &mut Option<SupervisedChild>,
15191    reason: String,
15192) -> Result<(), SuperviseError> {
15193    let now = Instant::now();
15194    let mut schedule = None;
15195    update_snapshot(snapshot, Some(&spec.module_id), |state| {
15196        clear_current_process_facts(state);
15197        if state.enabled {
15198            schedule = state.next_crash_restart(&runtime.restart_policy, now);
15199            state.state = if schedule.is_some() {
15200                ModuleState::Restarting
15201            } else {
15202                if runtime
15203                    .failure_context(runtime.recovery_is_daemon)
15204                    .may_retry()
15205                {
15206                    state.failed_retry.schedule(&spec.module_id);
15207                }
15208                ModuleState::Failed
15209            };
15210        } else {
15211            state.state = ModuleState::Disabled;
15212        }
15213    })?;
15214    if let Some(schedule) = schedule {
15215        schedule_respawn(
15216            runtime,
15217            snapshot,
15218            &spec.module_id,
15219            schedule.delay,
15220            RespawnKind::Spawn,
15221            true,
15222        )?;
15223    } else {
15224        process_liveness.untrack_if_current(&spec.module_id, snapshot);
15225    }
15226    Err(SuperviseError::ReloadFailed {
15227        module_id: spec.module_id.clone(),
15228        reason,
15229    })
15230}
15231
15232fn control_flags() -> Flags {
15233    Flags::new(false, Priority::Passive, false)
15234}
15235
15236#[allow(clippy::too_many_arguments)]
15237async fn drain_optional_child(
15238    module_id: &str,
15239    protocol: ModuleProtocol,
15240    stop_notice: StopNotice,
15241    registry: &Registry,
15242    forwarding: Option<&ForwardingTable>,
15243    snapshot: &SharedSnapshot,
15244    terminal_ring: &Arc<Mutex<TerminalRing>>,
15245    spawn_events: &SpawnEventFeed,
15246    child: &mut Option<SupervisedChild>,
15247    drain_timeout: Duration,
15248    final_state: ModuleState,
15249    enabled: Option<bool>,
15250    failure: &FailureContext,
15251) -> Result<(), SuperviseError> {
15252    if let Some(child) = child.take() {
15253        drain_child_to_state(
15254            module_id,
15255            protocol,
15256            stop_notice,
15257            registry,
15258            forwarding,
15259            snapshot,
15260            terminal_ring,
15261            spawn_events,
15262            child,
15263            drain_timeout,
15264            final_state,
15265            enabled,
15266            failure,
15267        )
15268        .await
15269    } else {
15270        update_snapshot(snapshot, Some(module_id), |state| {
15271            state.state = final_state;
15272            if final_state == ModuleState::Failed {
15273                state.failed_retry.failed_at = Some(Instant::now());
15274            }
15275            if let Some(enabled) = enabled {
15276                state.enabled = enabled;
15277            }
15278            clear_current_process_facts(state);
15279        })?;
15280        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
15281    }
15282}
15283
15284#[allow(clippy::too_many_arguments)]
15285async fn drain_child_to_state(
15286    module_id: &str,
15287    _protocol: ModuleProtocol,
15288    stop_notice: StopNotice,
15289    registry: &Registry,
15290    forwarding: Option<&ForwardingTable>,
15291    snapshot: &SharedSnapshot,
15292    terminal_ring: &Arc<Mutex<TerminalRing>>,
15293    spawn_events: &SpawnEventFeed,
15294    mut child: SupervisedChild,
15295    drain_timeout: Duration,
15296    final_state: ModuleState,
15297    enabled: Option<bool>,
15298    failure: &FailureContext,
15299) -> Result<(), SuperviseError> {
15300    let protocol = child.protocol;
15301    child.retire_registration();
15302    update_snapshot(snapshot, Some(module_id), |state| {
15303        state.state = ModuleState::Draining;
15304        state.draining_to_replace = final_state == ModuleState::Restarting;
15305        if let Some(enabled) = enabled {
15306            state.enabled = enabled;
15307        }
15308    })?;
15309
15310    // The wait below is the same budget in every case; what differs is
15311    // whether anything has ASKED the child to stop before it starts. Only a
15312    // forwarding drain that reached the module's registered connection has
15313    // (`module.draining`, then a module GOODBYE). Every other child was told
15314    // nothing: a `protocol: "none"` module, which never registers; a subc
15315    // module spawned moments ago that has not sent HELLO yet; or a stop that
15316    // runs no forwarding drain. Without a signal the budget is only a delay
15317    // in front of SIGKILL -- and the not-yet-registered child is the worst
15318    // case, because it registers into a module that is already draining,
15319    // is never told, and is killed while healthy.
15320    if stop_notice != StopNotice::SentOverConnection {
15321        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
15322            info!(
15323                module_id,
15324                pid = child.pid,
15325                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15326                "module has no connection yet; requesting stop by signal"
15327            );
15328        }
15329        request_graceful_stop(module_id, &child);
15330    }
15331
15332    let exit_report = match timeout(drain_timeout, child.wait()).await {
15333        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
15334        Ok(Err(source)) => {
15335            fail_after_wait_error(
15336                module_id,
15337                snapshot,
15338                terminal_ring,
15339                spawn_events,
15340                format!("failed to wait during drain: {source}"),
15341                failure,
15342            );
15343            return Err(SuperviseError::Wait {
15344                module_id: module_id.to_string(),
15345                source,
15346            });
15347        }
15348        Err(_) => {
15349            // Mirror the sibling arm above: state is already `Draining`, and an
15350            // error propagated from here would strand it there -- a state
15351            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
15352            // `Failed | Stopped`), leaving an operator Restart as the only exit.
15353            // `Failed` before `?` keeps the module operator-visible and
15354            // revivable. Trigger is an ESRCH race (process exits between the
15355            // drain timeout firing and the kill) or a post-kill wait failure
15356            // (issue #34).
15357            //
15358            // Logged because the kill is otherwise visible only as signal 9 in
15359            // the terminal ring, and the budget it follows can be long enough
15360            // that consumers see a stretch of refusals with no stated cause.
15361            warn!(
15362                module_id,
15363                pid = child.pid,
15364                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15365                reason = ?final_state,
15366                ?stop_notice,
15367                "drain budget expired before the module exited; killing it"
15368            );
15369            let kill =
15370                if final_state == ModuleState::Restarting && child.child.container().is_some() {
15371                    child.kill_leader()
15372                } else {
15373                    child.start_kill()
15374                };
15375            kill.map_err(|source| {
15376                fail_after_wait_error(
15377                    module_id,
15378                    snapshot,
15379                    terminal_ring,
15380                    spawn_events,
15381                    format!("failed to kill during drain: {source}"),
15382                    failure,
15383                );
15384                SuperviseError::Kill {
15385                    module_id: module_id.to_string(),
15386                    source,
15387                }
15388            })?;
15389            let status = child.wait().await.map_err(|source| {
15390                fail_after_wait_error(
15391                    module_id,
15392                    snapshot,
15393                    terminal_ring,
15394                    spawn_events,
15395                    format!("failed to wait after drain kill: {source}"),
15396                    failure,
15397                );
15398                SuperviseError::Wait {
15399                    module_id: module_id.to_string(),
15400                    source,
15401                }
15402            })?;
15403            classify_reaped_child_exit(snapshot, &child, &status)
15404        }
15405    };
15406
15407    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
15408    publish_after_terminal_record(
15409        module_id,
15410        || {
15411            record_terminal_with_detail(
15412                module_id,
15413                terminal_ring,
15414                spawn_events,
15415                &exit_report,
15416                terminal_disposition(final_state),
15417                detail,
15418            )
15419        },
15420        || {
15421            update_snapshot(snapshot, Some(module_id), |state| {
15422                state.state = final_state;
15423                if final_state == ModuleState::Failed {
15424                    state.failed_retry.failed_at = Some(Instant::now());
15425                }
15426                if let Some(enabled) = enabled {
15427                    state.enabled = enabled;
15428                }
15429                clear_current_process_facts(state);
15430                state.last_exit = Some(exit_report.clone());
15431                if exit_report.kind == ExitKind::DeliberateSeverance {
15432                    state.lifetime_restarts += 1;
15433                }
15434            })
15435        },
15436    )?;
15437    child.drain_stderr(module_id).await;
15438
15439    let released =
15440        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
15441    child
15442        .finish_exit(
15443            released.is_ok()
15444                && final_state == ModuleState::Restarting
15445                && child.spawn_failure.is_none(),
15446        )
15447        .await
15448        .map_err(|source| SuperviseError::Wait {
15449            module_id: module_id.to_owned(),
15450            source,
15451        })?;
15452    released
15453}
15454
15455/// Ask a child that nothing else has asked to stop, by signal.
15456///
15457/// A registered subc module is asked over its own connection: the drain sends
15458/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
15459/// module GOODBYE, and the module stops itself. A module that speaks no subc
15460/// wire receives none of that, and neither does a subc module that has not
15461/// registered yet, so for them the drain budget would be pure delay in front of
15462/// a SIGKILL -- and for a process with a store to flush (JetStream is the
15463/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
15464/// into a recovery on the next start.
15465///
15466/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
15467/// rule rather than an optimisation: that module's graceful stop is already
15468/// running by the time its child is drained, and a signal would race it.
15469///
15470/// Best-effort by construction. A child that has already exited is the ordinary
15471/// case rather than an error (the kill lands on a reaped or exiting pid), so a
15472/// failure is logged at debug and the wait-then-kill below still decides the
15473/// outcome.
15474#[cfg(unix)]
15475fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
15476    let Some(pid) = child
15477        .id()
15478        .and_then(|pid| i32::try_from(pid).ok())
15479        .and_then(rustix::process::Pid::from_raw)
15480    else {
15481        debug!(
15482            module_id,
15483            "no pid to signal for teardown; falling through to the drain wait"
15484        );
15485        return;
15486    };
15487    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
15488        Ok(()) => debug!(
15489            module_id,
15490            "sent SIGTERM to a module nothing else asked to stop"
15491        ),
15492        Err(err) => debug!(
15493            module_id,
15494            error = %err,
15495            "SIGTERM to module failed; the drain wait and kill still apply"
15496        ),
15497    }
15498}
15499
15500/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
15501/// Windows does offer need cooperation this supervisor cannot assume: a console
15502/// control event requires sharing a console with the child, and `WM_CLOSE`
15503/// requires the child to pump a message loop. A supervised server process does
15504/// neither, so there is nothing to send and teardown is the wait followed by the
15505/// kill. Emulating a signal here would mean inventing a stop protocol, which is
15506/// the thing `protocol: "none"` exists to avoid.
15507#[cfg(not(unix))]
15508fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
15509    debug!(
15510        module_id,
15511        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
15512    );
15513}
15514
15515fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
15516    match final_state {
15517        ModuleState::Stopped => TerminalDisposition::Stopped,
15518        ModuleState::Disabled => TerminalDisposition::Disabled,
15519        ModuleState::Restarting => TerminalDisposition::Restarting,
15520        ModuleState::Failed => TerminalDisposition::Failed,
15521        ModuleState::Starting
15522        | ModuleState::Running
15523        | ModuleState::Unresponsive
15524        | ModuleState::Draining => {
15525            unreachable!("terminal exits only finish in terminal or restarting states")
15526        }
15527    }
15528}
15529
15530/// Release a reaped child's registration before allowing another spawn.
15531///
15532/// EOF is not a process-lifetime signal: an inherited socket can stay open
15533/// indefinitely, and serial frame dispatch can be waiting on egress instead of
15534/// reading EOF. After the normal release grace, request connection close (which
15535/// cancels both reads and dispatch), then allow one more release grace for the
15536/// connection guard's forwarding cleanup. Never evict a different connection.
15537async fn release_dead_registration(
15538    registry: &Registry,
15539    forwarding: Option<&ForwardingTable>,
15540    snapshot: &SharedSnapshot,
15541    module_id: &str,
15542    failure: &FailureContext,
15543) -> Result<(), SuperviseError> {
15544    let result = async {
15545        let registration = registry
15546            .get_module(module_id)
15547            .map_err(SuperviseError::Registry)?;
15548        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
15549            Ok(()) => return Ok(()),
15550            Err(SuperviseError::RegistrationStillActive { .. }) => {}
15551            Err(err) => return Err(err),
15552        }
15553        let pid = lock_snapshot(snapshot)?.reaped_pid;
15554        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
15555            warn!(
15556                module_id,
15557                pid,
15558                connection_id = registration.connection_id.get(),
15559                "reaped module registration outlived release grace; closing dead connection"
15560            );
15561            forwarding.request_connection_close(
15562                registration.connection_id,
15563                CloseReason::new(
15564                    "supervised_process_reaped",
15565                    format!("module '{module_id}' pid {pid} exited"),
15566                ),
15567            );
15568            wait_for_slot_registration_release(
15569                registry,
15570                crate::registry::RegistrationSlot::Connection(registration.connection_id),
15571                REGISTRY_RELEASE_TIMEOUT,
15572            )
15573            .await?;
15574        }
15575        wait_for_registration_release(registry, module_id, Duration::ZERO).await
15576    }
15577    .await;
15578    if let Err(err) = &result {
15579        fail_snapshot(snapshot, Some(module_id), None, failure);
15580        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
15581    }
15582    result
15583}
15584
15585/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
15586/// plain stop or restart waits for before it spawns a replacement.
15587async fn wait_for_registration_release(
15588    registry: &Registry,
15589    module_id: &str,
15590    wait: Duration,
15591) -> Result<(), SuperviseError> {
15592    wait_for_slot_registration_release(
15593        registry,
15594        crate::registry::RegistrationSlot::Active(module_id),
15595        wait,
15596    )
15597    .await
15598}
15599
15600/// Wait for the registration in `slot` to go away.
15601///
15602/// Keyed on the slot rather than the bare module id because a successful swap
15603/// never empties the id's active slot (the promoted candidate is in it), so an
15604/// id-keyed wait for the incumbent's release would always time out. Draining a
15605/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
15606/// incumbent's connection instead.
15607async fn wait_for_slot_registration_release(
15608    registry: &Registry,
15609    slot: crate::registry::RegistrationSlot<'_>,
15610    wait: Duration,
15611) -> Result<(), SuperviseError> {
15612    let deadline = Instant::now() + wait;
15613    let mut release_events = registration_release_events().subscribe();
15614    let still_active = |registration: &crate::registry::ModuleRegistration| {
15615        SuperviseError::RegistrationStillActive {
15616            module_id: registration.manifest.module_id.clone(),
15617            waited: wait,
15618        }
15619    };
15620    loop {
15621        let _observed_generation = *release_events.borrow_and_update();
15622        let Some(registration) = registry
15623            .registration(slot)
15624            .map_err(SuperviseError::Registry)?
15625        else {
15626            return Ok(());
15627        };
15628
15629        let now = Instant::now();
15630        if now >= deadline {
15631            return Err(still_active(&registration));
15632        }
15633
15634        let remaining = deadline.saturating_duration_since(now);
15635        match timeout(remaining, release_events.changed()).await {
15636            Ok(Ok(())) | Ok(Err(_)) => {}
15637            Err(_) => return Err(still_active(&registration)),
15638        }
15639    }
15640}
15641
15642#[cfg(test)]
15643mod slot_registration_wait_tests {
15644    use super::*;
15645    use crate::registry::{ConnectionId, RegistrationSlot};
15646    use subc_protocol::manifest::ModuleManifest;
15647
15648    #[tokio::test]
15649    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
15650        let registry = Arc::new(Registry::default());
15651        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
15652        let runtime = supervisor.runtime_config();
15653        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
15654        let spec = ModuleSpec {
15655            script: None,
15656            module_id: "enable-stale-registration".to_string(),
15657            program: PathBuf::from("/missing/enable-retry-test"),
15658            args: Vec::new(),
15659            env: Vec::new(),
15660            reserved: false,
15661            reserved_prefixes: Vec::new(),
15662            protocol: ModuleProtocol::Subc,
15663            overlap: Default::default(),
15664            carry_containment: false,
15665        };
15666        let connection = ConnectionId::new(90);
15667        registry
15668            .register_with_control_ops(
15669                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
15670                1,
15671                connection,
15672                Vec::new(),
15673            )
15674            .unwrap();
15675        let mut child = None;
15676        let err = set_child_enabled(
15677            &spec,
15678            &runtime,
15679            &registry,
15680            &supervisor.process_liveness,
15681            &snapshot,
15682            &mut child,
15683            true,
15684            &runtime.failure_context(false),
15685        )
15686        .await
15687        .unwrap_err();
15688        assert!(matches!(
15689            err,
15690            SuperviseError::RegistrationStillActive { .. }
15691        ));
15692        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15693        assert!(child.is_none());
15694        registry.deregister_connection(connection).unwrap();
15695        let err = set_child_enabled(
15696            &spec,
15697            &runtime,
15698            &registry,
15699            &supervisor.process_liveness,
15700            &snapshot,
15701            &mut child,
15702            true,
15703            &runtime.failure_context(false),
15704        )
15705        .await
15706        .unwrap_err();
15707        assert!(
15708            matches!(err, SuperviseError::Spawn { .. }),
15709            "second enable must attempt a spawn: {err}"
15710        );
15711        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15712    }
15713
15714    const INCUMBENT: u64 = 1;
15715    const CANDIDATE: u64 = 2;
15716
15717    fn swapped_registry() -> Arc<Registry> {
15718        let registry = Arc::new(Registry::default());
15719        let manifest = ModuleManifest::builder("m", "0.1.0").build();
15720        registry
15721            .register_with_control_ops(
15722                manifest.clone(),
15723                1,
15724                ConnectionId::new(INCUMBENT),
15725                Vec::new(),
15726            )
15727            .unwrap();
15728        registry
15729            .register_candidate_with_control_ops(
15730                manifest,
15731                1,
15732                ConnectionId::new(CANDIDATE),
15733                Vec::new(),
15734            )
15735            .unwrap();
15736        registry
15737    }
15738
15739    /// After a promotion the id's active slot is held by the new process, so an
15740    /// id-keyed wait for the incumbent's release can never succeed; the
15741    /// connection-keyed wait completes as soon as the incumbent deregisters.
15742    #[tokio::test]
15743    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
15744        let registry = swapped_registry();
15745        registry.promote_candidate("m").unwrap().unwrap();
15746
15747        assert!(matches!(
15748            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
15749            Err(SuperviseError::RegistrationStillActive { .. })
15750        ));
15751
15752        // Still held while the incumbent's connection has not deregistered.
15753        assert!(matches!(
15754            wait_for_slot_registration_release(
15755                &registry,
15756                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15757                Duration::from_millis(50),
15758            )
15759            .await,
15760            Err(SuperviseError::RegistrationStillActive { .. })
15761        ));
15762
15763        let releaser = Arc::clone(&registry);
15764        let release = tokio::spawn(async move {
15765            sleep(Duration::from_millis(20)).await;
15766            releaser
15767                .deregister_connection(ConnectionId::new(INCUMBENT))
15768                .unwrap();
15769            notify_registration_release();
15770        });
15771        wait_for_slot_registration_release(
15772            &registry,
15773            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15774            Duration::from_secs(5),
15775        )
15776        .await
15777        .expect("the incumbent's own registration is released");
15778        release.await.unwrap();
15779        assert!(registry.get_module("m").unwrap().is_some());
15780    }
15781
15782    /// The candidate slot is waited on separately from the active slot: the
15783    /// incumbent's registration neither holds up nor stands in for it.
15784    #[tokio::test]
15785    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
15786        let registry = swapped_registry();
15787        assert!(matches!(
15788            wait_for_slot_registration_release(
15789                &registry,
15790                RegistrationSlot::Candidate("m"),
15791                Duration::from_millis(50),
15792            )
15793            .await,
15794            Err(SuperviseError::RegistrationStillActive { .. })
15795        ));
15796        registry
15797            .deregister_connection(ConnectionId::new(CANDIDATE))
15798            .unwrap();
15799        wait_for_slot_registration_release(
15800            &registry,
15801            RegistrationSlot::Candidate("m"),
15802            Duration::from_millis(50),
15803        )
15804        .await
15805        .expect("a candidate slot with no candidate is released");
15806        assert!(registry
15807            .registration(RegistrationSlot::Active("m"))
15808            .unwrap()
15809            .is_some());
15810    }
15811}
15812
15813fn classify_exit(status: &ExitStatus) -> ExitReport {
15814    ExitReport {
15815        kind: if status.success() {
15816            ExitKind::Clean
15817        } else {
15818            ExitKind::Crash
15819        },
15820        code: status.code(),
15821        signal: exit_signal(status),
15822        at_ms: unix_ms_now(),
15823    }
15824}
15825
15826/// The terminal record for a module whose `wait()` call itself errored (e.g. the
15827/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
15828/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
15829/// disposition still must be `Failed` so the terminal ring is not silently missing
15830/// an entry, matching what `fail_snapshot` records for this same arm.
15831fn wait_error_exit_report() -> ExitReport {
15832    ExitReport {
15833        kind: ExitKind::Crash,
15834        code: None,
15835        signal: None,
15836        at_ms: unix_ms_now(),
15837    }
15838}
15839
15840#[cfg(unix)]
15841fn exit_signal(status: &ExitStatus) -> Option<i32> {
15842    use std::os::unix::process::ExitStatusExt;
15843
15844    status.signal()
15845}
15846
15847#[cfg(not(unix))]
15848fn exit_signal(_status: &ExitStatus) -> Option<i32> {
15849    None
15850}
15851
15852/// Give an operator-touched module its full crash budget back.
15853///
15854/// Named for the counter it used to zero; it now empties the in-window ring,
15855/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
15856/// ledger of what happened survives every operator action.
15857fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
15858    update_snapshot(snapshot, Some(module_id), |state| {
15859        state.clear_crash_restarts();
15860    })
15861}
15862
15863fn set_running(
15864    snapshot: &SharedSnapshot,
15865    child: &SupervisedChild,
15866    module_id: &str,
15867    spawn_events: &SpawnEventFeed,
15868) -> Result<(), SuperviseError> {
15869    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
15870        module_id: Some(module_id.to_string()),
15871    })?;
15872    state.spawn_generation = spawn_events.emit_spawned(
15873        module_id,
15874        child
15875            .registration_spawn
15876            .as_ref()
15877            .map(|spawn| spawn.authority.spawn_number()),
15878        child.pid,
15879        child.spawned_at_ms,
15880    );
15881    if std::mem::take(&mut state.coalesced_restart_pending) {
15882        let generation = state.spawn_generation;
15883        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
15884    }
15885    state.drain_disposition_detail = None;
15886    state.spawn_failure = None;
15887    state.admission_refused = None;
15888    state.config_hold_parked = false;
15889    state.config_hold_daemon_initiated = false;
15890    state.config_hold_approval = None;
15891    state.legacy_registration_spawn =
15892        child.registration_spawn.is_some() && child.admission_base.is_none();
15893    state.registration_declaration = child.registration_declaration.clone();
15894    state.stop_reason = None;
15895    state.config_exit_failed_approval = None;
15896    state.waiting_for_previous_pid = None;
15897    state.waiting_for_previous_detail = None;
15898    // Every caller of this is a plain spawn, which always uses the primary key;
15899    // a promoted swap candidate sets the flag itself after this returns.
15900    state.in_alternate_slot = false;
15901    state.configuration_updated_since_spawn = false;
15902    state.spawned_protocol = Some(child.protocol);
15903    state.state = ModuleState::Running;
15904    state.approved_start_failure = None;
15905    state.enabled = true;
15906    state.process_alive = true;
15907    state.pid = child.id();
15908    state.subtree_target = child.subtree_target.clone();
15909    // Probe samples belong to one process. Keep module-level escalation and
15910    // late-answer history, but never charge a new child for its predecessor's misses.
15911    state.health.status = SupervisorHealthStatus::Unknown;
15912    state.last_probe_evidence = None;
15913    state.health.consecutive_failures = 0;
15914    state.health.detail = None;
15915    state.health.metrics = None;
15916    state.health.last_probe_ms = None;
15917    state.failed_retry.healthy_since = None;
15918    state.failed_retry.failed_at = None;
15919    state.failed_retry.running_since = Some(child.spawned_at);
15920    #[cfg(windows)]
15921    {
15922        state.pipe_consumed = child
15923            .nonce_delivery
15924            .as_ref()
15925            .map(|delivery| delivery.consumed());
15926    }
15927    #[cfg(target_os = "macos")]
15928    {
15929        state.report_ready = Some(Arc::clone(&child.report_ready));
15930    }
15931    state.spawned_at_ms = Some(child.spawned_at_ms);
15932    state.exec_outcome = Some(Arc::clone(&child.exec.0));
15933    state.spawned_from = Some(child.spawned_from.clone());
15934    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
15935    state.spawned_file_identity = child.spawned_file_identity.clone();
15936    state.process_start_time = child.process_start_time;
15937    Ok(())
15938}
15939
15940fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
15941    state.legacy_registration_spawn = false;
15942    state.registration_declaration = None;
15943    state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
15944    state.failed_retry.healthy_since = None;
15945    state.failed_retry.running_since = None;
15946    state.waiting_for_previous_pid = None;
15947    state.waiting_for_previous_detail = None;
15948    state.process_alive = false;
15949    state.spawned_protocol = None;
15950    state.pid = None;
15951    #[cfg(windows)]
15952    {
15953        state.pipe_consumed = None;
15954    }
15955    #[cfg(target_os = "macos")]
15956    {
15957        state.report_ready = None;
15958    }
15959    state.spawned_at_ms = None;
15960    state.exec_outcome = None;
15961    state.spawned_from = None;
15962    state.launch_spec_digest = None;
15963    state.spawned_file_identity = None;
15964    state.process_start_time = None;
15965    state.deliberate_severance = None;
15966}
15967
15968#[cfg(test)]
15969fn record_deliberate_severance(
15970    snapshot: &SharedSnapshot,
15971    identity: ProcessIdentity,
15972) -> Result<(), SuperviseError> {
15973    update_snapshot(snapshot, None, |state| {
15974        state.deliberate_severance = Some(identity);
15975    })
15976}
15977
15978fn apply_deliberate_severance_marker(
15979    snapshot: &SharedSnapshot,
15980    exited_identity: Option<ProcessIdentity>,
15981    mut exit_report: ExitReport,
15982) -> ExitReport {
15983    let marker = lock_snapshot(snapshot)
15984        .ok()
15985        .and_then(|mut state| state.deliberate_severance.take());
15986    if marker.is_some() && marker == exited_identity {
15987        exit_report.kind = ExitKind::DeliberateSeverance;
15988    }
15989    exit_report
15990}
15991
15992fn classify_reaped_child_exit(
15993    snapshot: &SharedSnapshot,
15994    child: &SupervisedChild,
15995    status: &ExitStatus,
15996) -> ExitReport {
15997    let _ = update_snapshot(snapshot, None, |state| {
15998        state.reaped_pid = Some(child.pid);
15999        state.spawn_failure = child.spawn_failure.clone();
16000    });
16001    if matches!(
16002        child.spawn_failure.as_deref(),
16003        Some("registration_timeout" | "registration_channel_closed")
16004    ) {
16005        registration_failure_exit_report(classify_exit(status))
16006    } else {
16007        apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
16008    }
16009}
16010
16011fn fail_snapshot(
16012    snapshot: &SharedSnapshot,
16013    module_id: Option<&str>,
16014    last_exit: Option<ExitReport>,
16015    failure: &FailureContext,
16016) {
16017    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
16018        state.state = ModuleState::Failed;
16019        state.config_hold_parked = false;
16020        state.config_hold_daemon_initiated = false;
16021        state.config_hold_approval = None;
16022        clear_current_process_facts(state);
16023        if failure.may_retry() && state.enabled {
16024            state.failed_retry.schedule(module_id.unwrap_or("unknown"));
16025        } else if failure.shutdown.is_set() {
16026            state.failed_retry.cancel();
16027        }
16028        if let Some(last_exit) = last_exit {
16029            state.last_exit = Some(last_exit);
16030        }
16031    }) {
16032        error!(error = %err, "failed to mark supervisor state failed");
16033    }
16034}
16035
16036fn update_snapshot(
16037    snapshot: &SharedSnapshot,
16038    module_id: Option<&str>,
16039    update: impl FnOnce(&mut SupervisorSnapshot),
16040) -> Result<(), SuperviseError> {
16041    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
16042        module_id: module_id.map(ToOwned::to_owned),
16043    })?;
16044    update(&mut state);
16045    Ok(())
16046}
16047
16048const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
16049
16050fn lock_snapshot_for_control<'a>(
16051    snapshot: &'a SharedSnapshot,
16052    module_id: &str,
16053    caller: &'static str,
16054) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
16055    let started_at = Instant::now();
16056    let guard = lock_snapshot(snapshot)?;
16057    let waited = started_at.elapsed();
16058    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
16059        warn!(
16060            module_id = %module_id,
16061            waited_ms = waited.as_millis() as u64,
16062            caller = %caller,
16063            "slow snapshot lock"
16064        );
16065    }
16066    Ok(guard)
16067}
16068
16069fn lock_snapshot(
16070    snapshot: &SharedSnapshot,
16071) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
16072    snapshot
16073        .lock()
16074        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
16075}
16076
16077#[cfg(test)]
16078mod terminal_history_tests {
16079    include!("supervise_route_open_recovery_tests.rs");
16080    use std::{
16081        path::PathBuf,
16082        sync::Arc,
16083        time::{Duration, Instant},
16084    };
16085
16086    use tokio::{sync::mpsc, time::sleep};
16087
16088    use super::{
16089        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
16090        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
16091        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
16092        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
16093        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
16094        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
16095        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
16096        SupervisorHealthStatus, SupervisorSnapshot,
16097    };
16098    // The supervisor's clock, distinct from the `std::time::Instant` these tests
16099    // use for their own wall-clock deadlines: crash-restart instants must be on
16100    // the same clock the production code stamps them with, which is tokio's (and
16101    // is what `start_paused` tests can move).
16102    use super::Instant as ClockInstant;
16103    use crate::{
16104        registry::Registry,
16105        terminal_ring::{TerminalRing, TerminalRingConfig},
16106    };
16107    use std::sync::Mutex;
16108    use subc_control::TerminalDisposition;
16109
16110    /// See the twin in `control.rs` for why this derives the path from
16111    /// `current_exe()` and why the existence check is here: `--lib` alone does
16112    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
16113    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
16114    pub(super) fn fake_aft_stub_path() -> PathBuf {
16115        let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
16116        assert!(
16117            path.exists(),
16118            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
16119             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
16120            path.display()
16121        );
16122        path
16123    }
16124
16125    #[test]
16126    fn reserved_never_spawned_refuses_every_hello() {
16127        // The canary hole: a reserved id whose module has never spawned had NO
16128        // gate entry and admitted anyone -- the reservation protected the nonce
16129        // holder, not the NAME. Now the entry is present with no legitimate
16130        // holder and refuses all comers.
16131        let supervisor = SupervisorHandle::default();
16132        supervisor.apply_identity_configuration(&ModuleSpec {
16133            script: None,
16134            module_id: "never-spawned".to_string(),
16135            program: PathBuf::from("/usr/bin/false"),
16136            args: Vec::new(),
16137            env: Vec::new(),
16138            reserved: true,
16139            reserved_prefixes: Vec::new(),
16140            protocol: ModuleProtocol::Subc,
16141            overlap: Default::default(),
16142            carry_containment: false,
16143        });
16144        assert!(
16145            supervisor
16146                .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
16147                .is_err(),
16148            "forged nonce must refuse on a reserved never-spawned id"
16149        );
16150        assert!(
16151            supervisor
16152                .reserved_hello_verification("never-spawned", None)
16153                .is_err(),
16154            "absent nonce must refuse on a reserved never-spawned id"
16155        );
16156        // And a real spawn nonce minted later admits exactly that nonce.
16157        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
16158        supervisor.apply_identity_configuration(&ModuleSpec {
16159            script: None,
16160            module_id: "never-spawned".to_string(),
16161            program: PathBuf::from("/usr/bin/false"),
16162            args: Vec::new(),
16163            env: Vec::new(),
16164            reserved: true,
16165            reserved_prefixes: Vec::new(),
16166            protocol: ModuleProtocol::Subc,
16167            overlap: Default::default(),
16168            carry_containment: false,
16169        });
16170        assert!(supervisor
16171            .reserved_hello_verification("never-spawned", Some("minted"))
16172            .is_ok());
16173        assert!(supervisor
16174            .reserved_hello_verification("never-spawned", Some("forged"))
16175            .is_err());
16176    }
16177
16178    /// Put `count` crash restarts on a snapshot's ring as if they had all just
16179    /// happened, which is what "spent budget" looks like to every reader.
16180    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
16181        let now = ClockInstant::now();
16182        for _ in 0..count {
16183            state.crash_restarts.push_back(now);
16184        }
16185    }
16186
16187    /// Age the oldest recorded restart out of `window`, standing in for the hours
16188    /// that would otherwise have to pass. Injecting the instant is the point: a
16189    /// test that slept a real window would take ten minutes and still prove less.
16190    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
16191        let aged = state
16192            .crash_restarts
16193            .front()
16194            .expect("a crash restart must be recorded before it can be aged")
16195            .checked_sub(window + Duration::from_secs(1))
16196            .expect("the test clock is far enough from its origin to age an instant");
16197        state.crash_restarts[0] = aged;
16198    }
16199
16200    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
16201        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
16202        seed_crash_restarts(&mut state, count);
16203        state
16204    }
16205
16206    #[test]
16207    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
16208        let policy = RestartPolicy::new(3, Duration::ZERO);
16209        let now = ClockInstant::now();
16210        assert!(daemon_will_restart(
16211            &mut snapshot_with_restarts(true, 2),
16212            &policy,
16213            now
16214        ));
16215        assert!(!daemon_will_restart(
16216            &mut snapshot_with_restarts(true, 3),
16217            &policy,
16218            now
16219        ));
16220        assert!(!daemon_will_restart(
16221            &mut snapshot_with_restarts(false, 0),
16222            &policy,
16223            now
16224        ));
16225    }
16226
16227    #[test]
16228    fn crash_restart_backoff_escalates_with_in_window_count() {
16229        let policy = RestartPolicy::new(4, Duration::from_millis(100))
16230            .with_max_backoff(Duration::from_secs(30));
16231        let now = ClockInstant::now();
16232        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16233        let schedules = (0..4)
16234            .map(|_| {
16235                state
16236                    .next_crash_restart(&policy, now)
16237                    .expect("the test policy allows four crash restarts")
16238            })
16239            .collect::<Vec<_>>();
16240
16241        assert_eq!(
16242            schedules
16243                .iter()
16244                .map(|schedule| schedule.restart_in_window)
16245                .collect::<Vec<_>>(),
16246            vec![0, 1, 2, 3]
16247        );
16248        assert_eq!(
16249            schedules
16250                .iter()
16251                .map(|schedule| schedule.delay)
16252                .collect::<Vec<_>>(),
16253            vec![
16254                Duration::from_millis(100),
16255                Duration::from_secs(1),
16256                Duration::from_secs(10),
16257                Duration::from_secs(30),
16258            ]
16259        );
16260    }
16261
16262    #[test]
16263    fn crash_restart_backoff_resets_after_ring_clear() {
16264        let policy = RestartPolicy::new(3, Duration::from_millis(100));
16265        let now = ClockInstant::now();
16266        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16267        assert_eq!(
16268            state.next_crash_restart(&policy, now).unwrap().delay,
16269            Duration::from_millis(100)
16270        );
16271        assert_eq!(
16272            state.next_crash_restart(&policy, now).unwrap().delay,
16273            Duration::from_secs(1)
16274        );
16275
16276        state.clear_crash_restarts();
16277        let schedule = state
16278            .next_crash_restart(&policy, now)
16279            .expect("a cleared ring must allow another restart");
16280        assert_eq!(schedule.restart_in_window, 0);
16281        assert_eq!(schedule.delay, Duration::from_millis(100));
16282    }
16283
16284    #[test]
16285    fn crash_restart_backoff_ignores_aged_restarts() {
16286        let policy = RestartPolicy::new(3, Duration::from_millis(100));
16287        let now = ClockInstant::now();
16288        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16289        state
16290            .next_crash_restart(&policy, now)
16291            .expect("the first restart is allowed");
16292        state
16293            .next_crash_restart(&policy, now)
16294            .expect("the second restart is allowed");
16295        state.crash_restarts[0] = now
16296            .checked_sub(policy.window + Duration::from_secs(1))
16297            .expect("the fake clock can age a restart past the window");
16298
16299        let schedule = state
16300            .next_crash_restart(&policy, now)
16301            .expect("an aged restart must release its slot");
16302        assert_eq!(schedule.restart_in_window, 1);
16303        assert_eq!(schedule.delay, Duration::from_secs(1));
16304        assert_eq!(state.crash_restarts.len(), 2);
16305    }
16306
16307    /// The budget is a rate: the same three spent restarts refuse a respawn
16308    /// while they are recent and allow one once they have aged past the window.
16309    /// Nothing about the module changed in between, which is the whole point.
16310    #[test]
16311    fn a_budget_spent_before_the_window_no_longer_refuses() {
16312        let policy = RestartPolicy::new(3, Duration::ZERO);
16313        let mut state = snapshot_with_restarts(true, 3);
16314        let now = ClockInstant::now();
16315        assert!(!daemon_will_restart(&mut state, &policy, now));
16316
16317        assert!(daemon_will_restart(
16318            &mut state,
16319            &policy,
16320            now + policy.window + Duration::from_secs(1)
16321        ));
16322        assert!(
16323            state.crash_restarts.is_empty(),
16324            "reading the budget must drop the instants that left the window"
16325        );
16326    }
16327
16328    fn module_with_recovery_snapshot(
16329        state: ModuleState,
16330        enabled: bool,
16331        restart_count: u32,
16332    ) -> SupervisedModule {
16333        let registry = Arc::new(Registry::default());
16334        let supervisor =
16335            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
16336        let runtime = supervisor.runtime_config();
16337        let spec = ModuleSpec {
16338            script: None,
16339            module_id: "recovery-snapshot".to_string(),
16340            program: fake_aft_stub_path(),
16341            args: Vec::new(),
16342            env: Vec::new(),
16343            reserved: false,
16344            reserved_prefixes: Vec::new(),
16345            protocol: ModuleProtocol::Subc,
16346            overlap: Default::default(),
16347            carry_containment: false,
16348        };
16349        let mut snapshot = SupervisorSnapshot::new(state, enabled);
16350        seed_crash_restarts(&mut snapshot, restart_count);
16351        // These tests read synthetic snapshots. A real child and monitor would
16352        // race those reads by replacing the requested state during startup.
16353        let (commands, _rx) = mpsc::channel(4);
16354        let configured_enabled = snapshot.enabled;
16355        SupervisedModule {
16356            inner: Arc::new(SupervisedModuleInner {
16357                module_id: spec.module_id.clone(),
16358                registry,
16359                snapshot: Arc::new(Mutex::new(snapshot)),
16360                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
16361                    spec,
16362                    enabled: configured_enabled,
16363                    health: runtime.health,
16364                    register_timeout: runtime.register_timeout,
16365                    restart_policy: runtime.restart_policy,
16366                })),
16367                stderr_ring: runtime.stderr_ring,
16368                terminal_ring: runtime.terminal_ring,
16369                commands,
16370                monitor: Mutex::new(None),
16371                effective_drain_timeout: runtime.effective_drain_timeout,
16372                provenance_probe: supervisor.provenance_probe.clone(),
16373                retained: runtime.child_roster.retained_view(),
16374            }),
16375        }
16376    }
16377
16378    #[cfg(target_os = "linux")]
16379    #[tokio::test]
16380    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
16381        let supervisor =
16382            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16383                .with_cgroup_placement(None);
16384        let result = supervisor.spawn(ModuleSpec {
16385            script: None,
16386            module_id: "no-cgroup-placement".to_string(),
16387            program: fake_aft_stub_path(),
16388            args: Vec::new(),
16389            env: Vec::new(),
16390            reserved: false,
16391            reserved_prefixes: Vec::new(),
16392            protocol: ModuleProtocol::Subc,
16393            overlap: Default::default(),
16394            carry_containment: false,
16395        });
16396
16397        assert!(
16398            result.is_ok(),
16399            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
16400        );
16401    }
16402
16403    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16404    async fn undecided_snapshot_uses_shared_restart_predicate() {
16405        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
16406            .will_recover_after_connection_loss()
16407            .unwrap());
16408        assert!(
16409            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
16410                .will_recover_after_connection_loss()
16411                .unwrap()
16412        );
16413    }
16414
16415    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16416    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
16417        assert!(
16418            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
16419                .will_recover_after_connection_loss()
16420                .unwrap()
16421        );
16422    }
16423
16424    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16425    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
16426        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
16427            .will_recover_after_connection_loss()
16428            .unwrap());
16429        assert!(
16430            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
16431                .will_recover_after_connection_loss()
16432                .unwrap()
16433        );
16434    }
16435
16436    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16437    async fn warming_snapshot_is_limited_to_startup_phases() {
16438        for state in [
16439            ModuleState::Starting,
16440            ModuleState::Running,
16441            ModuleState::Restarting,
16442        ] {
16443            assert!(
16444                module_with_recovery_snapshot(state, true, 0)
16445                    .is_warming()
16446                    .unwrap(),
16447                "{state:?} should be warming"
16448            );
16449        }
16450        for state in [
16451            ModuleState::Unresponsive,
16452            ModuleState::Draining,
16453            ModuleState::Stopped,
16454            ModuleState::Failed,
16455            ModuleState::Disabled,
16456        ] {
16457            assert!(
16458                !module_with_recovery_snapshot(state, true, 0)
16459                    .is_warming()
16460                    .unwrap(),
16461                "{state:?} should not be warming"
16462            );
16463        }
16464    }
16465
16466    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16467    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
16468        let registry = Arc::new(Registry::default());
16469        let supervisor =
16470            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
16471        let module = supervisor
16472            .spawn(ModuleSpec {
16473                script: None,
16474                module_id: "terminal-history".to_string(),
16475                program: fake_aft_stub_path(),
16476                args: Vec::new(),
16477                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16478                reserved: false,
16479                reserved_prefixes: Vec::new(),
16480                protocol: ModuleProtocol::Subc,
16481                overlap: Default::default(),
16482                carry_containment: false,
16483            })
16484            .unwrap();
16485
16486        let deadline = Instant::now() + Duration::from_secs(5);
16487        loop {
16488            let history = module.terminal_history();
16489            if history.entries.len() == 2 {
16490                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
16491                assert_eq!(history.dropped, 0);
16492                assert_eq!(
16493                    history
16494                        .entries
16495                        .iter()
16496                        .map(|entry| entry.exit_code)
16497                        .collect::<Vec<_>>(),
16498                    vec![Some(23), Some(23)]
16499                );
16500                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
16501                return;
16502            }
16503            assert!(
16504                Instant::now() < deadline,
16505                "module did not retain two terminal exits: {history:?}"
16506            );
16507            sleep(Duration::from_millis(10)).await;
16508        }
16509    }
16510
16511    /// A disable issued while a crash respawn is still backing off must preempt
16512    /// that respawn: the operator's stop wins, the disable must not queue behind
16513    /// the backoff, and the module must never come back up afterwards.
16514    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16515    async fn disable_during_crash_backoff_cancels_pending_respawn() {
16516        let backoff = Duration::from_secs(2);
16517        let supervisor = Supervisor::new_for_test(
16518            Arc::new(Registry::default()),
16519            RestartPolicy::new(10, backoff),
16520        );
16521        let module = supervisor
16522            .spawn(ModuleSpec {
16523                script: None,
16524                module_id: "disable-during-backoff".to_string(),
16525                program: fake_aft_stub_path(),
16526                args: Vec::new(),
16527                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16528                reserved: false,
16529                reserved_prefixes: Vec::new(),
16530                protocol: ModuleProtocol::Subc,
16531                overlap: Default::default(),
16532                carry_containment: false,
16533            })
16534            .unwrap();
16535
16536        // Wait for the first crash to put the module into its backoff window.
16537        let deadline = Instant::now() + Duration::from_secs(5);
16538        loop {
16539            if module.status().unwrap().state == ModuleState::Restarting {
16540                break;
16541            }
16542            assert!(
16543                Instant::now() < deadline,
16544                "module never entered the crash backoff"
16545            );
16546            sleep(Duration::from_millis(10)).await;
16547        }
16548
16549        let started = Instant::now();
16550        module.set_enabled(false).await.unwrap();
16551        let waited = started.elapsed();
16552
16553        assert!(
16554            waited < backoff / 2,
16555            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
16556        );
16557        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
16558
16559        // Outlast the backoff: the respawn it was counting down to must never run.
16560        sleep(backoff + Duration::from_millis(500)).await;
16561        let status = module.status().unwrap();
16562        assert_eq!(status.state, ModuleState::Disabled);
16563        assert_eq!(
16564            status.spawn_generation, 1,
16565            "module respawned after the operator disabled it"
16566        );
16567    }
16568
16569    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
16570    /// the shape of nats-server, the program this rule exists for.
16571    #[cfg(unix)]
16572    fn protocol_none_sigterm_exits_clean_spec(
16573        module_id: &str,
16574        dir: &std::path::Path,
16575    ) -> (ModuleSpec, PathBuf, PathBuf) {
16576        let ready = dir.join("ready");
16577        let marker = dir.join("sigterm");
16578        let spec = ModuleSpec {
16579            script: None,
16580            module_id: module_id.to_string(),
16581            program: fake_aft_stub_path(),
16582            args: Vec::new(),
16583            env: vec![
16584                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16585                (
16586                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
16587                    marker.display().to_string(),
16588                ),
16589                (
16590                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
16591                    ready.display().to_string(),
16592                ),
16593            ],
16594            reserved: false,
16595            reserved_prefixes: Vec::new(),
16596            protocol: ModuleProtocol::None,
16597            overlap: Default::default(),
16598            carry_containment: false,
16599        };
16600        (spec, ready, marker)
16601    }
16602
16603    /// Wait for a file the child writes, so a signal is never sent before the
16604    /// child's SIGTERM handler is installed (the default disposition would
16605    /// kill it by signal and the exit would not be clean).
16606    #[cfg(unix)]
16607    async fn wait_for_file(path: &std::path::Path) {
16608        let deadline = Instant::now() + Duration::from_secs(10);
16609        while !path.exists() {
16610            assert!(
16611                Instant::now() < deadline,
16612                "{} never appeared",
16613                path.display()
16614            );
16615            sleep(Duration::from_millis(10)).await;
16616        }
16617    }
16618
16619    /// A protocol-none module that exits 0 because something OUTSIDE the
16620    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
16621    /// the crash-path disposition rather than `stopped`.
16622    #[cfg(unix)]
16623    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16624    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
16625        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
16626        let (spec, ready, marker) =
16627            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
16628        let supervisor = Supervisor::new_for_test(
16629            Arc::new(Registry::default()),
16630            RestartPolicy::new(3, Duration::ZERO),
16631        );
16632        let module = supervisor.spawn(spec).unwrap();
16633        wait_for_file(&ready).await;
16634        // The ready file proves the child installed its SIGTERM handler, not
16635        // that the supervisor has processed the privacy trampoline's exec
16636        // acknowledgement. On macOS status withholds the pid until then.
16637        let deadline = Instant::now() + Duration::from_secs(10);
16638        let first_pid = loop {
16639            let status = module.status().unwrap();
16640            if status.state == ModuleState::Running {
16641                if let Some(pid) = status.pid {
16642                    break pid;
16643                }
16644            }
16645            assert!(
16646                Instant::now() < deadline,
16647                "a running module must report its pid after exec confirmation: {status:?}"
16648            );
16649            sleep(Duration::from_millis(10)).await;
16650        };
16651
16652        rustix::process::kill_process(
16653            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
16654            rustix::process::Signal::TERM,
16655        )
16656        .unwrap();
16657
16658        let deadline = Instant::now() + Duration::from_secs(10);
16659        let respawned = loop {
16660            let status = module.status().unwrap();
16661            if status.state == ModuleState::Running
16662                && status.pid.is_some_and(|pid| pid != first_pid)
16663            {
16664                break status;
16665            }
16666            assert!(
16667                Instant::now() < deadline,
16668                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
16669            );
16670            sleep(Duration::from_millis(10)).await;
16671        };
16672        assert_eq!(respawned.spawn_generation, 2);
16673        assert!(
16674            marker.exists(),
16675            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
16676        );
16677
16678        let history = module.terminal_history();
16679        assert_eq!(history.entries.len(), 1, "{history:?}");
16680        let entry = &history.entries[0];
16681        assert_eq!(entry.exit_code, Some(0));
16682        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
16683        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
16684
16685        module.stop().await.unwrap();
16686    }
16687
16688    /// Repeated unrequested clean exits of a protocol-none module spend the
16689    /// restart budget exactly as crashes do, and the module ends `failed` with
16690    /// the budget named.
16691    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16692    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
16693        let supervisor = Supervisor::new_for_test(
16694            Arc::new(Registry::default()),
16695            RestartPolicy::new(1, Duration::ZERO),
16696        );
16697        let module = supervisor
16698            .spawn(ModuleSpec {
16699                script: None,
16700                module_id: "none-clean-exit-budget".to_string(),
16701                program: fake_aft_stub_path(),
16702                args: Vec::new(),
16703                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
16704                reserved: false,
16705                reserved_prefixes: Vec::new(),
16706                protocol: ModuleProtocol::None,
16707                overlap: Default::default(),
16708                carry_containment: false,
16709            })
16710            .unwrap();
16711
16712        // Failed follows the terminal write; this deadline only bounds a hang,
16713        // not an assumed duration for the two launches or their exit recording.
16714        let deadline = Instant::now() + Duration::from_secs(10);
16715        loop {
16716            let status = module.status().unwrap();
16717            if status.state == ModuleState::Failed {
16718                break;
16719            }
16720            assert!(
16721                Instant::now() < deadline,
16722                "module never exhausted its budget: {status:?} {:?}",
16723                module.terminal_history()
16724            );
16725            sleep(Duration::from_millis(10)).await;
16726        }
16727        let history = module.terminal_history();
16728        assert_eq!(
16729            history
16730                .entries
16731                .iter()
16732                .map(|entry| (entry.exit_code, entry.disposition.clone()))
16733                .collect::<Vec<_>>(),
16734            vec![
16735                (Some(0), TerminalDisposition::Restarting),
16736                (Some(0), TerminalDisposition::Failed),
16737            ]
16738        );
16739        let detail = history.entries[1]
16740            .disposition_detail
16741            .as_deref()
16742            .expect("a budget failure names the budget");
16743        assert!(detail.contains("max_restarts=1"), "{detail}");
16744        assert_eq!(module.status().unwrap().spawn_generation, 2);
16745    }
16746
16747    #[test]
16748    fn restart_budget_failure_is_published_after_its_terminal_record() {
16749        use std::io::{Read, Write};
16750        let supervisor = Supervisor::new_for_test(
16751            Arc::new(Registry::default()),
16752            RestartPolicy::new(0, Duration::ZERO),
16753        );
16754        let runtime = supervisor.runtime_config();
16755        let spec = ModuleSpec {
16756            script: None,
16757            protocol: ModuleProtocol::None,
16758            ..windowed_crash_spec("budget-publication-order")
16759        };
16760        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
16761            ModuleState::Running,
16762            true,
16763        )));
16764        lock_snapshot(&snapshot).unwrap().process_alive = true;
16765        let reaped_snapshot = snapshot.clone();
16766        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16767        let history = runtime.terminal_ring.clone();
16768        let reap = std::thread::spawn(move || {
16769            tokio::runtime::Builder::new_current_thread()
16770                .enable_all()
16771                .build()
16772                .unwrap()
16773                .block_on(on_child_exit(
16774                    &spec,
16775                    runtime.restart_policy,
16776                    &supervisor.registry,
16777                    &reaped_snapshot,
16778                    &runtime.terminal_ring,
16779                    &runtime.spawn_events,
16780                    &runtime.child_roster,
16781                    ExitReport {
16782                        kind: ExitKind::Clean,
16783                        code: Some(0),
16784                        signal: None,
16785                        at_ms: 1,
16786                    },
16787                    true,
16788                ))
16789        });
16790        let mut reached = [0];
16791        barrier.read_exact(&mut reached).unwrap();
16792        assert_eq!(&reached, b"R");
16793        let (before_state, before_alive) = {
16794            let state = lock_snapshot(&snapshot).unwrap();
16795            (state.state, state.process_alive)
16796        };
16797        let before_history = history.lock().unwrap().snapshot();
16798        barrier.write_all(b"X").unwrap();
16799        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
16800        assert!(before_history.entries.is_empty());
16801        assert_ne!(
16802            before_state,
16803            ModuleState::Failed,
16804            "Failed was visible before its terminal record could be written"
16805        );
16806        assert!(before_alive);
16807        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
16808        let history = history.lock().unwrap().snapshot();
16809        assert_eq!(history.entries.len(), 1);
16810        assert_eq!(history.entries[0].exit_code, Some(0));
16811        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16812    }
16813
16814    #[test]
16815    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
16816        use std::io::{Read, Write};
16817
16818        let supervisor = Supervisor::new_for_test(
16819            Arc::new(Registry::default()),
16820            RestartPolicy::new(3, Duration::ZERO),
16821        );
16822        let mut runtime = supervisor.runtime_config();
16823        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
16824        let snapshot = module.inner.snapshot.clone();
16825        lock_snapshot(&snapshot).unwrap().process_alive = true;
16826        runtime.terminal_ring = module.inner.terminal_ring.clone();
16827        let spec = windowed_crash_spec(module.module_id());
16828        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16829        let reaped_snapshot = snapshot.clone();
16830        let reap = std::thread::spawn(move || {
16831            tokio::runtime::Builder::new_current_thread()
16832                .enable_all()
16833                .build()
16834                .unwrap()
16835                .block_on(on_child_exit(
16836                    &spec,
16837                    runtime.restart_policy,
16838                    &supervisor.registry,
16839                    &reaped_snapshot,
16840                    &runtime.terminal_ring,
16841                    &runtime.spawn_events,
16842                    &runtime.child_roster,
16843                    ExitReport {
16844                        kind: ExitKind::Crash,
16845                        code: Some(2),
16846                        signal: None,
16847                        at_ms: 1,
16848                    },
16849                    true,
16850                ))
16851        });
16852
16853        // Hold the crash handler between its budget decision and publication.
16854        // Connection cleanup must still see the recovery that this crash chose,
16855        // even when that recovery consumes the last available restart slot.
16856        let mut reached = [0];
16857        barrier.read_exact(&mut reached).unwrap();
16858        assert_eq!(&reached, b"R");
16859        let before_status = module.status().unwrap();
16860        let before_history = module.terminal_history();
16861        let before_recovers = module.will_recover_after_connection_loss().unwrap();
16862        barrier.write_all(b"X").unwrap();
16863        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
16864
16865        assert_eq!(before_status.state, ModuleState::Running);
16866        assert!(before_status.process_alive);
16867        assert_eq!(before_status.restart_count, 3);
16868        assert!(before_history.entries.is_empty());
16869        assert!(
16870            before_recovers,
16871            "connection cleanup would report a terminal close for a scheduled crash restart"
16872        );
16873        let restarting = module.status().unwrap();
16874        assert_eq!(restarting.state, ModuleState::Restarting);
16875        assert_eq!(restarting.restart_count, 3);
16876        assert!(module.will_recover_after_connection_loss().unwrap());
16877        let history = module.terminal_history();
16878        assert_eq!(history.entries.len(), 1);
16879        assert_eq!(
16880            history.entries[0].disposition,
16881            TerminalDisposition::Restarting
16882        );
16883
16884        // Once the replacement is running, its next crash must be judged
16885        // against the exhausted budget rather than the previous recovery.
16886        update_snapshot(&snapshot, Some(module.module_id()), |state| {
16887            state.state = ModuleState::Running;
16888            state.respawn_pending = false;
16889        })
16890        .unwrap();
16891        assert!(!module.will_recover_after_connection_loss().unwrap());
16892        update_snapshot(&snapshot, Some(module.module_id()), |state| {
16893            state.enabled = false;
16894            state.respawn_pending = true;
16895        })
16896        .unwrap();
16897        assert!(!module.will_recover_after_connection_loss().unwrap());
16898    }
16899
16900    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
16901        use std::net::{TcpListener, TcpStream};
16902        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
16903        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
16904        let (barrier, _) = listener.accept().unwrap();
16905        for stream in [&writer, &barrier] {
16906            stream
16907                .set_read_timeout(Some(Duration::from_secs(10)))
16908                .unwrap();
16909            stream
16910                .set_write_timeout(Some(Duration::from_secs(10)))
16911                .unwrap();
16912        }
16913        *events.1.lock().unwrap() = Some(writer);
16914        barrier
16915    }
16916
16917    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
16918        use std::io::{Read, Write};
16919        let barrier = events.1.lock().unwrap().take();
16920        if let Some(mut barrier) = barrier {
16921            barrier.write_all(b"R").unwrap();
16922            let mut release = [0];
16923            barrier.read_exact(&mut release).unwrap();
16924            assert_eq!(&release, b"X");
16925        }
16926    }
16927
16928    #[test]
16929    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
16930        use std::io::{Read, Write};
16931
16932        let supervisor = Supervisor::new_for_test(
16933            Arc::new(Registry::default()),
16934            RestartPolicy::new(0, Duration::ZERO),
16935        );
16936        let mut runtime = supervisor.runtime_config();
16937        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
16938        let snapshot = module.inner.snapshot.clone();
16939        runtime.terminal_ring = module.inner.terminal_ring.clone();
16940        let spec = ModuleSpec {
16941            script: None,
16942            module_id: module.module_id().into(),
16943            program: fake_aft_stub_path(),
16944            protocol: ModuleProtocol::None,
16945            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
16946            ..windowed_crash_spec(module.module_id())
16947        };
16948        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16949        let reap = std::thread::spawn(move || {
16950            tokio::runtime::Builder::new_current_thread()
16951                .enable_all()
16952                .build()
16953                .unwrap()
16954                .block_on(async {
16955                    let mut child =
16956                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
16957                    health_restart_child(
16958                        &spec,
16959                        &runtime,
16960                        &supervisor.registry,
16961                        &supervisor.process_liveness,
16962                        &snapshot,
16963                        &mut child,
16964                        SupervisorHealthStatus::Failing,
16965                        Some("health probe did not answer"),
16966                        1,
16967                    )
16968                    .await
16969                    .unwrap();
16970                    assert!(child.is_none());
16971                });
16972        });
16973        // The loopback barrier acknowledges entry into terminal recording by
16974        // health_restart_child's drain, then waits for the test's release byte.
16975        // Status and history are observed while that writer is held, not after
16976        // an elapsed-time guess about when the health action has completed.
16977        let mut reached = [0];
16978        barrier.read_exact(&mut reached).unwrap();
16979        assert_eq!(&reached, b"R");
16980        let before_status = module.status().unwrap();
16981        let before_history = module.terminal_history();
16982        barrier.write_all(b"X").unwrap();
16983        reap.join().unwrap();
16984        assert!(before_history.entries.is_empty());
16985        assert_ne!(
16986            before_status.state,
16987            ModuleState::Failed,
16988            "Failed was visible before its terminal record could be written"
16989        );
16990        assert!(
16991            before_status.process_alive,
16992            "process death was visible before its terminal record could be written"
16993        );
16994        let failed = module.status().unwrap();
16995        assert_eq!(failed.state, ModuleState::Failed);
16996        assert!(!failed.process_alive);
16997        let history = module.terminal_history();
16998        assert_eq!(history.entries.len(), 1);
16999        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
17000        assert_eq!(
17001            history.entries[0].disposition_detail.as_deref(),
17002            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
17003        );
17004    }
17005
17006    #[test]
17007    fn observed_exits_publish_process_death_after_terminal_history() {
17008        use std::io::{Read, Write};
17009        for (kind, enabled, shutdown, wait_error, expected_state) in [
17010            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
17011            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
17012            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
17013            (
17014                ExitKind::DeliberateSeverance,
17015                true,
17016                false,
17017                false,
17018                ModuleState::Restarting,
17019            ),
17020            #[cfg(unix)]
17021            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
17022            (ExitKind::Crash, true, false, true, ModuleState::Failed),
17023        ] {
17024            let supervisor = Supervisor::new_for_test(
17025                Arc::new(Registry::default()),
17026                RestartPolicy::new(1, Duration::ZERO),
17027            );
17028            let runtime = supervisor.runtime_config();
17029            #[cfg(unix)]
17030            if shutdown {
17031                runtime.child_roster.close();
17032            }
17033            #[cfg(not(unix))]
17034            let _ = shutdown;
17035            let spec = windowed_crash_spec("exit-publication-order");
17036            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
17037                ModuleState::Running,
17038                enabled,
17039            )));
17040            lock_snapshot(&snapshot).unwrap().process_alive = true;
17041            let reaped_snapshot = snapshot.clone();
17042            let history = runtime.terminal_ring.clone();
17043            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
17044            let reap = std::thread::spawn(move || {
17045                if wait_error {
17046                    super::fail_after_wait_error(
17047                        &spec.module_id,
17048                        &reaped_snapshot,
17049                        &runtime.terminal_ring,
17050                        &runtime.spawn_events,
17051                        "wait failed".into(),
17052                        &runtime.failure_context(true),
17053                    );
17054                } else {
17055                    tokio::runtime::Builder::new_current_thread()
17056                        .enable_all()
17057                        .build()
17058                        .unwrap()
17059                        .block_on(on_child_exit(
17060                            &spec,
17061                            runtime.restart_policy,
17062                            &supervisor.registry,
17063                            &reaped_snapshot,
17064                            &runtime.terminal_ring,
17065                            &runtime.spawn_events,
17066                            &runtime.child_roster,
17067                            ExitReport {
17068                                kind,
17069                                code: Some(0),
17070                                signal: None,
17071                                at_ms: 1,
17072                            },
17073                            true,
17074                        ));
17075                }
17076            });
17077            let mut reached = [0];
17078            barrier.read_exact(&mut reached).unwrap();
17079            assert_eq!(&reached, b"R");
17080            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
17081            let before_history = history.lock().unwrap().snapshot();
17082            barrier.write_all(b"X").unwrap();
17083            reap.join().unwrap();
17084            assert!(
17085                before_alive,
17086                "{kind:?}: death was published before recording"
17087            );
17088            assert!(before_history.entries.is_empty());
17089            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
17090            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
17091            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
17092        }
17093    }
17094
17095    #[tokio::test]
17096    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
17097        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
17098        let path = dir.join("terminals.jsonl");
17099        std::fs::create_dir(&path).unwrap();
17100        let supervisor = Supervisor::new_for_test(
17101            Arc::new(Registry::default()),
17102            RestartPolicy::new(0, Duration::ZERO),
17103        )
17104        .with_terminal_journal(path, "health-budget-journal-failure".into());
17105        let runtime = supervisor.runtime_config();
17106        let spec = ModuleSpec {
17107            script: None,
17108            program: fake_aft_stub_path(),
17109            protocol: ModuleProtocol::None,
17110            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
17111            ..windowed_crash_spec("health-budget-journal-failure")
17112        };
17113        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17114        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
17115        health_restart_child(
17116            &spec,
17117            &runtime,
17118            &supervisor.registry,
17119            &supervisor.process_liveness,
17120            &snapshot,
17121            &mut child,
17122            SupervisorHealthStatus::Failing,
17123            None,
17124            1,
17125        )
17126        .await
17127        .unwrap();
17128        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17129        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
17130        let history = runtime
17131            .terminal_ring
17132            .lock()
17133            .unwrap()
17134            .durable_history(&spec.module_id);
17135        assert!(history.journal_write_failures > 0);
17136        assert_eq!(history.entries.len(), 1);
17137        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
17138    }
17139
17140    #[test]
17141    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
17142        let supervisor = Supervisor::new_for_test(
17143            Arc::new(Registry::default()),
17144            RestartPolicy::new(0, Duration::ZERO),
17145        );
17146        let runtime = supervisor.runtime_config();
17147        let spec = ModuleSpec {
17148            script: None,
17149            program: fake_aft_stub_path(),
17150            protocol: ModuleProtocol::None,
17151            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
17152            ..windowed_crash_spec("health-budget-recording-panic")
17153        };
17154        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17155        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
17156            tokio::runtime::Builder::new_current_thread()
17157                .enable_all()
17158                .build()
17159                .unwrap()
17160                .block_on(async {
17161                    let mut child =
17162                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
17163                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
17164                    health_restart_child(
17165                        &spec,
17166                        &runtime,
17167                        &supervisor.registry,
17168                        &supervisor.process_liveness,
17169                        &snapshot,
17170                        &mut child,
17171                        SupervisorHealthStatus::Failing,
17172                        None,
17173                        1,
17174                    )
17175                    .await
17176                    .unwrap();
17177                });
17178        }));
17179        let panic = result.expect_err("recording must still unwind");
17180        assert_eq!(
17181            panic.downcast_ref::<String>().map(String::as_str),
17182            Some("spawn event sequence exhausted")
17183        );
17184        let state = lock_snapshot(&snapshot).unwrap();
17185        assert_eq!(state.state, ModuleState::Failed);
17186        assert!(!state.process_alive);
17187        assert!(state.last_exit.is_some());
17188    }
17189
17190    #[tokio::test]
17191    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
17192        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
17193        let path = dir.join("terminals.jsonl");
17194        std::fs::create_dir(&path).unwrap();
17195        let supervisor = Supervisor::new_for_test(
17196            Arc::new(Registry::default()),
17197            RestartPolicy::new(0, Duration::ZERO),
17198        )
17199        .with_terminal_journal(path, "budget-journal-failure".into());
17200        let runtime = supervisor.runtime_config();
17201        let spec = windowed_crash_spec("budget-journal-failure");
17202        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17203        assert!(matches!(
17204            on_child_exit(
17205                &spec,
17206                runtime.restart_policy,
17207                &supervisor.registry,
17208                &snapshot,
17209                &runtime.terminal_ring,
17210                &runtime.spawn_events,
17211                &runtime.child_roster,
17212                crash_exit_report(1),
17213                true,
17214            )
17215            .await,
17216            NextAction::Stop { .. }
17217        ));
17218        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17219        let history = runtime
17220            .terminal_ring
17221            .lock()
17222            .unwrap()
17223            .durable_history(&spec.module_id);
17224        assert!(history.journal_write_failures > 0);
17225        assert_eq!(history.entries.len(), 1);
17226        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
17227    }
17228
17229    #[test]
17230    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
17231        let supervisor = Supervisor::new_for_test(
17232            Arc::new(Registry::default()),
17233            RestartPolicy::new(0, Duration::ZERO),
17234        );
17235        let runtime = supervisor.runtime_config();
17236        let spec = windowed_crash_spec("budget-recording-panic");
17237        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17238        runtime
17239            .spawn_events
17240            .emit_spawned(&spec.module_id, None, 1, 1);
17241        // Exhausting the event sequence makes emit_exited panic before the
17242        // terminal write, exercising publication on the recording unwind.
17243        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
17244        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
17245            tokio::runtime::Builder::new_current_thread()
17246                .enable_all()
17247                .build()
17248                .unwrap()
17249                .block_on(on_child_exit(
17250                    &spec,
17251                    runtime.restart_policy,
17252                    &supervisor.registry,
17253                    &snapshot,
17254                    &runtime.terminal_ring,
17255                    &runtime.spawn_events,
17256                    &runtime.child_roster,
17257                    crash_exit_report(1),
17258                    true,
17259                ))
17260        }));
17261        let panic = result.err().expect("recording must still unwind");
17262        assert_eq!(
17263            panic.downcast_ref::<String>().map(String::as_str),
17264            Some("spawn event sequence exhausted")
17265        );
17266        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17267    }
17268
17269    /// A stop the supervisor itself requests still stops a protocol-none
17270    /// module, even though the child answers the SIGTERM with exit 0.
17271    #[cfg(unix)]
17272    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17273    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
17274        for disable in [false, true] {
17275            let label = if disable {
17276                "none-requested-disable"
17277            } else {
17278                "none-requested-stop"
17279            };
17280            let dir = cortexkit_test_support::ScratchDir::new(label);
17281            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
17282            let supervisor = Supervisor::new_for_test(
17283                Arc::new(Registry::default()),
17284                RestartPolicy::new(3, Duration::ZERO),
17285            );
17286            let module = supervisor.spawn(spec).unwrap();
17287            wait_for_file(&ready).await;
17288
17289            if disable {
17290                module.set_enabled(false).await.unwrap();
17291            } else {
17292                module.stop().await.unwrap();
17293            }
17294            assert!(
17295                marker.exists(),
17296                "{label}: the child must have left through its SIGTERM handler with exit 0"
17297            );
17298
17299            // Long enough for a zero-backoff respawn to have happened if the
17300            // exit had been treated as a crash.
17301            sleep(Duration::from_millis(500)).await;
17302            let status = module.status().unwrap();
17303            let expected = if disable {
17304                ModuleState::Disabled
17305            } else {
17306                ModuleState::Stopped
17307            };
17308            assert_eq!(status.state, expected, "{label}");
17309            assert_eq!(
17310                status.spawn_generation, 1,
17311                "{label}: respawned after a requested stop"
17312            );
17313            let history = module.terminal_history();
17314            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
17315            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
17316            assert_ne!(
17317                history.entries[0].disposition,
17318                TerminalDisposition::Restarting,
17319                "{label}"
17320            );
17321        }
17322    }
17323
17324    /// A subc-wire module that exits 0 on its own is still a stop: the
17325    /// protocol-none rule must not reach it.
17326    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17327    async fn subc_wire_clean_exit_is_still_a_stop() {
17328        let supervisor = Supervisor::new_for_test(
17329            Arc::new(Registry::default()),
17330            RestartPolicy::new(3, Duration::ZERO),
17331        );
17332        let module = supervisor
17333            .spawn(ModuleSpec {
17334                script: None,
17335                module_id: "wire-clean-exit".to_string(),
17336                program: fake_aft_stub_path(),
17337                args: Vec::new(),
17338                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
17339                reserved: false,
17340                reserved_prefixes: Vec::new(),
17341                protocol: ModuleProtocol::Subc,
17342                overlap: Default::default(),
17343                carry_containment: false,
17344            })
17345            .unwrap();
17346
17347        let deadline = Instant::now() + Duration::from_secs(10);
17348        while module.terminal_history().entries.is_empty() {
17349            assert!(Instant::now() < deadline, "module never exited");
17350            sleep(Duration::from_millis(10)).await;
17351        }
17352        // Long enough for a zero-backoff respawn to have happened.
17353        sleep(Duration::from_millis(500)).await;
17354        let status = module.status().unwrap();
17355        assert_eq!(status.state, ModuleState::Stopped);
17356        assert_eq!(status.spawn_generation, 1);
17357        let history = module.terminal_history();
17358        assert_eq!(history.entries.len(), 1, "{history:?}");
17359        assert_eq!(history.entries[0].exit_code, Some(0));
17360        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
17361    }
17362
17363    #[cfg(unix)]
17364    #[tokio::test]
17365    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
17366        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
17367        let record = dir.join("live-children.json");
17368        let supervisor = Supervisor::new_for_test(
17369            Arc::new(Registry::default()),
17370            RestartPolicy::new(0, Duration::ZERO),
17371        );
17372        let mut runtime = supervisor.runtime_config();
17373        runtime.child_roster.record_to(record.clone());
17374        let gate = Arc::new(super::ReloadExitRecordGate::default());
17375        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
17376        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17377        let spec = ModuleSpec {
17378            script: None,
17379            module_id: "reload-exit-roster".into(),
17380            program: fake_aft_stub_path(),
17381            args: Vec::new(),
17382            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
17383            reserved: false,
17384            reserved_prefixes: Vec::new(),
17385            protocol: ModuleProtocol::Subc,
17386            overlap: Default::default(),
17387            carry_containment: false,
17388        };
17389        let mut child = None;
17390        let reload = super::finish_reload_child(
17391            &spec,
17392            &runtime,
17393            &supervisor.registry,
17394            &supervisor.process_liveness,
17395            &snapshot,
17396            &mut child,
17397        );
17398        tokio::pin!(reload);
17399        tokio::select! {
17400            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
17401            _ = gate.reached.notified() => {}
17402        }
17403        assert!(runtime
17404            .terminal_ring
17405            .lock()
17406            .unwrap()
17407            .snapshot()
17408            .entries
17409            .is_empty());
17410        assert_eq!(
17411            crate::live_children::read_record(&record).unwrap().len(),
17412            1,
17413            "shutdown must still wait for the reaped child until its terminal record exists"
17414        );
17415        runtime.child_roster.close();
17416        gate.resume.notify_one();
17417        assert!(reload.await.is_err());
17418        assert!(crate::live_children::read_record(&record)
17419            .unwrap()
17420            .is_empty());
17421        let history = runtime.terminal_ring.lock().unwrap().snapshot();
17422        assert_eq!(history.entries.len(), 1);
17423        assert_eq!(
17424            history.entries[0].disposition,
17425            TerminalDisposition::DaemonShutdown
17426        );
17427    }
17428
17429    /// Each restart-producing arm has its own state transition. Keeping their
17430    /// lifetime count assertions adjacent prevents a later new arm from silently
17431    /// spending budget without recording the historical restart.
17432    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17433    async fn every_restart_increment_path_advances_lifetime_count() {
17434        let supervisor = Supervisor::new_for_test(
17435            Arc::new(Registry::default()),
17436            RestartPolicy::new(1, Duration::ZERO),
17437        );
17438        let runtime = supervisor.runtime_config();
17439        let spec = ModuleSpec {
17440            script: None,
17441            module_id: "lifetime-increment-path".to_string(),
17442            program: PathBuf::from("/unused/lifetime-increment-path"),
17443            args: Vec::new(),
17444            env: Vec::new(),
17445            reserved: false,
17446            reserved_prefixes: Vec::new(),
17447            protocol: ModuleProtocol::Subc,
17448            overlap: Default::default(),
17449            carry_containment: false,
17450        };
17451
17452        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17453        assert!(matches!(
17454            on_child_exit(
17455                &spec,
17456                runtime.restart_policy,
17457                &supervisor.registry,
17458                &crash_snapshot,
17459                &runtime.terminal_ring,
17460                &runtime.spawn_events,
17461                &runtime.child_roster,
17462                ExitReport {
17463                    kind: ExitKind::Crash,
17464                    code: Some(1),
17465                    signal: None,
17466                    at_ms: 1,
17467                },
17468                true,
17469            )
17470            .await,
17471            NextAction::Restart { schedule: _ }
17472        ));
17473        let (crash_restarts, crash_lifetime) = {
17474            let state = lock_snapshot(&crash_snapshot).unwrap();
17475            (state.crash_restarts.len(), state.lifetime_restarts)
17476        };
17477        assert_eq!(crash_restarts, 1);
17478        assert_eq!(crash_lifetime, 1);
17479
17480        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17481        let mut health_child = None;
17482        assert!(matches!(
17483            health_restart_child(
17484                &spec,
17485                &runtime,
17486                &supervisor.registry,
17487                &supervisor.process_liveness,
17488                &health_snapshot,
17489                &mut health_child,
17490                SupervisorHealthStatus::Failing,
17491                None,
17492                2,
17493            )
17494            .await,
17495            Ok(())
17496        ));
17497        assert!(health_child.is_none());
17498        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
17499        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
17500        let (health_restarts, health_lifetime) = {
17501            let state = lock_snapshot(&health_snapshot).unwrap();
17502            (state.crash_restarts.len(), state.lifetime_restarts)
17503        };
17504        assert_eq!(health_restarts, 1);
17505        assert_eq!(health_lifetime, 1);
17506
17507        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17508        let mut reload_child = None;
17509        assert!(matches!(
17510            handle_reload_spawn_failure(
17511                &spec,
17512                &runtime,
17513                &supervisor.process_liveness,
17514                &reload_snapshot,
17515                &mut reload_child,
17516                "forced reload spawn failure".to_string(),
17517            )
17518            .await,
17519            Err(SuperviseError::ReloadFailed { .. })
17520        ));
17521        let (reload_restarts, reload_lifetime) = {
17522            let state = lock_snapshot(&reload_snapshot).unwrap();
17523            (state.crash_restarts.len(), state.lifetime_restarts)
17524        };
17525        assert_eq!(reload_restarts, 1);
17526        assert_eq!(reload_lifetime, 1);
17527    }
17528
17529    #[tokio::test]
17530    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
17531        let supervisor = Supervisor::new_for_test(
17532            Arc::new(Registry::default()),
17533            RestartPolicy::new(3, Duration::ZERO),
17534        );
17535        let runtime = supervisor.runtime_config();
17536        let spec = ModuleSpec {
17537            script: None,
17538            module_id: "deliberately-severed".to_string(),
17539            program: PathBuf::from("/unused/deliberately-severed"),
17540            args: Vec::new(),
17541            env: Vec::new(),
17542            reserved: false,
17543            reserved_prefixes: Vec::new(),
17544            protocol: ModuleProtocol::Subc,
17545            overlap: Default::default(),
17546            carry_containment: false,
17547        };
17548        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17549        let process = ProcessIdentity {
17550            pid: 41,
17551            start_time: 101,
17552        };
17553        record_deliberate_severance(&snapshot, process).unwrap();
17554        let exit_report = apply_deliberate_severance_marker(
17555            &snapshot,
17556            Some(process),
17557            ExitReport {
17558                kind: ExitKind::Crash,
17559                code: Some(1),
17560                signal: None,
17561                at_ms: 1,
17562            },
17563        );
17564        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
17565
17566        assert!(matches!(
17567            on_child_exit(
17568                &spec,
17569                runtime.restart_policy,
17570                &supervisor.registry,
17571                &snapshot,
17572                &runtime.terminal_ring,
17573                &runtime.spawn_events,
17574                &runtime.child_roster,
17575                exit_report,
17576                true,
17577            )
17578            .await,
17579            NextAction::Restart { schedule: _ }
17580        ));
17581        let state = lock_snapshot(&snapshot).unwrap();
17582        assert_eq!(state.lifetime_restarts, 1);
17583        assert_eq!(state.crash_restarts.len(), 0);
17584    }
17585
17586    #[tokio::test]
17587    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
17588        let supervisor = Supervisor::new_for_test(
17589            Arc::new(Registry::default()),
17590            RestartPolicy::new(3, Duration::ZERO),
17591        );
17592        let runtime = supervisor.runtime_config();
17593        let spec = ModuleSpec {
17594            script: None,
17595            module_id: "genuine-crash".to_string(),
17596            program: PathBuf::from("/unused/genuine-crash"),
17597            args: Vec::new(),
17598            env: Vec::new(),
17599            reserved: false,
17600            reserved_prefixes: Vec::new(),
17601            protocol: ModuleProtocol::Subc,
17602            overlap: Default::default(),
17603            carry_containment: false,
17604        };
17605        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17606
17607        assert!(matches!(
17608            on_child_exit(
17609                &spec,
17610                runtime.restart_policy,
17611                &supervisor.registry,
17612                &snapshot,
17613                &runtime.terminal_ring,
17614                &runtime.spawn_events,
17615                &runtime.child_roster,
17616                ExitReport {
17617                    kind: ExitKind::Crash,
17618                    code: Some(1),
17619                    signal: None,
17620                    at_ms: 1,
17621                },
17622                true,
17623            )
17624            .await,
17625            NextAction::Restart { schedule: _ }
17626        ));
17627        let state = lock_snapshot(&snapshot).unwrap();
17628        assert_eq!(state.lifetime_restarts, 1);
17629        assert_eq!(state.crash_restarts.len(), 1);
17630    }
17631
17632    fn crash_exit_report(at_ms: u64) -> ExitReport {
17633        ExitReport {
17634            kind: ExitKind::Crash,
17635            code: Some(1),
17636            signal: None,
17637            at_ms,
17638        }
17639    }
17640
17641    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
17642        ModuleSpec {
17643            script: None,
17644            module_id: module_id.to_string(),
17645            program: PathBuf::from("/unused").join(module_id),
17646            args: Vec::new(),
17647            env: Vec::new(),
17648            reserved: false,
17649            reserved_prefixes: Vec::new(),
17650            protocol: ModuleProtocol::Subc,
17651            overlap: Default::default(),
17652            carry_containment: false,
17653        }
17654    }
17655
17656    /// A real crash loop still stops. Three crashes with nothing aging out spend
17657    /// a budget of two and the third respawn is refused, and both surfaces an
17658    /// operator has -- the log line and the retained terminal record -- name the
17659    /// window rather than only the cap, because `max_restarts=2` alone is what
17660    /// this budget used to mean.
17661    #[tokio::test]
17662    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
17663        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
17664        let supervisor = Supervisor::new_for_test(
17665            Arc::new(Registry::default()),
17666            RestartPolicy::new(2, Duration::ZERO),
17667        );
17668        let runtime = supervisor.runtime_config();
17669        let spec = windowed_crash_spec("crash-loop-in-window");
17670        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17671
17672        for attempt in 1..=2 {
17673            assert!(
17674                matches!(
17675                    on_child_exit(
17676                        &spec,
17677                        runtime.restart_policy,
17678                        &supervisor.registry,
17679                        &snapshot,
17680                        &runtime.terminal_ring,
17681                        &runtime.spawn_events,
17682                        &runtime.child_roster,
17683                        crash_exit_report(attempt),
17684                        true,
17685                    )
17686                    .await,
17687                    NextAction::Restart { schedule: _ }
17688                ),
17689                "crash {attempt} is inside the budget and must respawn"
17690            );
17691        }
17692
17693        assert!(matches!(
17694            on_child_exit(
17695                &spec,
17696                runtime.restart_policy,
17697                &supervisor.registry,
17698                &snapshot,
17699                &runtime.terminal_ring,
17700                &runtime.spawn_events,
17701                &runtime.child_roster,
17702                crash_exit_report(3),
17703                true,
17704            )
17705            .await,
17706            NextAction::Stop { .. }
17707        ));
17708
17709        {
17710            let state = lock_snapshot(&snapshot).unwrap();
17711            assert_eq!(state.state, ModuleState::Failed);
17712            assert_eq!(state.crash_restarts.len(), 2);
17713            assert_eq!(state.lifetime_restarts, 2);
17714        }
17715
17716        let history = runtime
17717            .terminal_ring
17718            .lock()
17719            .expect("terminal ring is not poisoned")
17720            .snapshot();
17721        let last = history
17722            .entries
17723            .last()
17724            .expect("the refused crash is retained");
17725        assert_eq!(last.disposition, TerminalDisposition::Failed);
17726        assert_eq!(
17727            last.disposition_detail.as_deref(),
17728            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
17729        );
17730
17731        let captured = crate::router::test_log::captured_logs(&logs);
17732        assert!(
17733            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
17734            "the stop must be logged with its window: {captured}"
17735        );
17736    }
17737
17738    /// The rate, stated as a test: three crashes where the first has aged past
17739    /// the window are two crashes as far as the budget is concerned, so the
17740    /// third respawn is allowed and the ring holds only the two recent ones.
17741    ///
17742    /// This is the case a lifetime counter got wrong -- and the case the daemon
17743    /// now hits routinely, since a module exits non-zero every time its
17744    /// connection to the daemon drops.
17745    #[tokio::test]
17746    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
17747        let supervisor = Supervisor::new_for_test(
17748            Arc::new(Registry::default()),
17749            RestartPolicy::new(2, Duration::ZERO),
17750        );
17751        let runtime = supervisor.runtime_config();
17752        let spec = windowed_crash_spec("crash-across-windows");
17753        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17754
17755        for attempt in 1..=2 {
17756            assert!(matches!(
17757                on_child_exit(
17758                    &spec,
17759                    runtime.restart_policy,
17760                    &supervisor.registry,
17761                    &snapshot,
17762                    &runtime.terminal_ring,
17763                    &runtime.spawn_events,
17764                    &runtime.child_roster,
17765                    crash_exit_report(attempt),
17766                    true,
17767                )
17768                .await,
17769                NextAction::Restart { schedule: _ }
17770            ));
17771        }
17772
17773        // The oldest crash moves out of the window; nothing else about the
17774        // module changes.
17775        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17776            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
17777        })
17778        .unwrap();
17779
17780        assert!(
17781            matches!(
17782                on_child_exit(
17783                    &spec,
17784                    runtime.restart_policy,
17785                    &supervisor.registry,
17786                    &snapshot,
17787                    &runtime.terminal_ring,
17788                    &runtime.spawn_events,
17789                    &runtime.child_roster,
17790                    crash_exit_report(3),
17791                    true,
17792                )
17793                .await,
17794                NextAction::Restart { schedule: _ }
17795            ),
17796            "a crash older than the window must not hold a budget slot"
17797        );
17798
17799        let state = lock_snapshot(&snapshot).unwrap();
17800        assert_eq!(state.state, ModuleState::Restarting);
17801        assert_eq!(
17802            state.crash_restarts.len(),
17803            2,
17804            "the aged instant is dropped and the new one takes its place"
17805        );
17806        assert_eq!(
17807            state.lifetime_restarts, 3,
17808            "the ledger counts every restart, including the ones the window forgot"
17809        );
17810    }
17811
17812    /// An operator restart hands the budget back whole, and the ledger keeps
17813    /// counting. Those are different questions -- "how close is this module to
17814    /// being stopped" and "how many times has it been replaced" -- and the
17815    /// operator action answers only the first.
17816    #[tokio::test]
17817    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
17818        let supervisor = Supervisor::new_for_test(
17819            Arc::new(Registry::default()),
17820            RestartPolicy::new(2, Duration::ZERO),
17821        );
17822        let runtime = supervisor.runtime_config();
17823        let spec = windowed_crash_spec("operator-cleared-budget");
17824        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17825
17826        for attempt in 1..=2 {
17827            assert!(matches!(
17828                on_child_exit(
17829                    &spec,
17830                    runtime.restart_policy,
17831                    &supervisor.registry,
17832                    &snapshot,
17833                    &runtime.terminal_ring,
17834                    &runtime.spawn_events,
17835                    &runtime.child_roster,
17836                    crash_exit_report(attempt),
17837                    true,
17838                )
17839                .await,
17840                NextAction::Restart { schedule: _ }
17841            ));
17842        }
17843
17844        reset_restart_count(&snapshot, &spec.module_id).unwrap();
17845        {
17846            let state = lock_snapshot(&snapshot).unwrap();
17847            assert!(
17848                state.crash_restarts.is_empty(),
17849                "an operator restart returns the full budget"
17850            );
17851            assert_eq!(
17852                state.lifetime_restarts, 2,
17853                "clearing the budget must not unmake the crashes"
17854            );
17855        }
17856
17857        assert!(
17858            matches!(
17859                on_child_exit(
17860                    &spec,
17861                    runtime.restart_policy,
17862                    &supervisor.registry,
17863                    &snapshot,
17864                    &runtime.terminal_ring,
17865                    &runtime.spawn_events,
17866                    &runtime.child_roster,
17867                    crash_exit_report(3),
17868                    true,
17869                )
17870                .await,
17871                NextAction::Restart { schedule: _ }
17872            ),
17873            "the cleared budget must be spendable again"
17874        );
17875        let state = lock_snapshot(&snapshot).unwrap();
17876        assert_eq!(state.crash_restarts.len(), 1);
17877        assert_eq!(state.lifetime_restarts, 3);
17878    }
17879
17880    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17881    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
17882        let severed = ProcessIdentity {
17883            pid: 41,
17884            start_time: 101,
17885        };
17886        let successor = ProcessIdentity {
17887            pid: 41,
17888            start_time: 202,
17889        };
17890        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
17891        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
17892            state.pid = Some(successor.pid);
17893            state.process_start_time = Some(successor.start_time);
17894        })
17895        .unwrap();
17896        assert!(!module.record_deliberate_severance(severed).unwrap());
17897
17898        let exit_report = apply_deliberate_severance_marker(
17899            &module.inner.snapshot,
17900            Some(successor),
17901            ExitReport {
17902                kind: ExitKind::Crash,
17903                code: Some(1),
17904                signal: None,
17905                at_ms: 1,
17906            },
17907        );
17908
17909        assert_eq!(exit_report.kind, ExitKind::Crash);
17910    }
17911
17912    #[tokio::test]
17913    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
17914        let registry = Registry::default();
17915        let supervisor = Supervisor::new_for_test(
17916            Arc::new(Registry::default()),
17917            RestartPolicy::new(3, Duration::ZERO),
17918        );
17919        let runtime = supervisor.runtime_config();
17920        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17921        let spec = ModuleSpec {
17922            script: None,
17923            module_id: "drain-deliberate-severance".to_string(),
17924            program: fake_aft_stub_path(),
17925            args: Vec::new(),
17926            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17927            reserved: false,
17928            reserved_prefixes: Vec::new(),
17929            protocol: ModuleProtocol::Subc,
17930            overlap: Default::default(),
17931            carry_containment: false,
17932        };
17933        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17934        let process = ProcessIdentity {
17935            pid: 41,
17936            start_time: 101,
17937        };
17938        child.process_identity = Some(process);
17939        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17940            state.pid = Some(process.pid);
17941            state.process_start_time = Some(process.start_time);
17942        })
17943        .unwrap();
17944        record_deliberate_severance(&snapshot, process).unwrap();
17945
17946        drain_child_to_state(
17947            &spec.module_id,
17948            spec.protocol,
17949            // The child exits on its own; no signal may change the exit this
17950            // test classifies.
17951            StopNotice::SentOverConnection,
17952            &registry,
17953            None,
17954            &snapshot,
17955            &runtime.terminal_ring,
17956            &runtime.spawn_events,
17957            child,
17958            Duration::from_secs(1),
17959            ModuleState::Stopped,
17960            Some(false),
17961            &super::FailureContext::operator(),
17962        )
17963        .await
17964        .unwrap();
17965
17966        let state = lock_snapshot(&snapshot).unwrap();
17967        assert_eq!(
17968            state.last_exit.as_ref().map(|exit| exit.kind),
17969            Some(ExitKind::DeliberateSeverance)
17970        );
17971        assert_eq!(state.lifetime_restarts, 1);
17972        assert_eq!(state.crash_restarts.len(), 0);
17973        drop(state);
17974        let history = runtime.terminal_ring.lock().unwrap().snapshot();
17975        assert_eq!(
17976            history.entries[0].exit_kind,
17977            subc_control::TerminalExitKind::DeliberateSeverance
17978        );
17979    }
17980
17981    #[tokio::test]
17982    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
17983        let registry = Registry::default();
17984        let supervisor = Supervisor::new_for_test(
17985            Arc::new(Registry::default()),
17986            RestartPolicy::new(3, Duration::ZERO),
17987        );
17988        let runtime = supervisor.runtime_config();
17989        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17990        let spec = ModuleSpec {
17991            script: None,
17992            module_id: "ordinary-drain".to_string(),
17993            program: fake_aft_stub_path(),
17994            args: Vec::new(),
17995            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17996            reserved: false,
17997            reserved_prefixes: Vec::new(),
17998            protocol: ModuleProtocol::Subc,
17999            overlap: Default::default(),
18000            carry_containment: false,
18001        };
18002        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
18003
18004        drain_child_to_state(
18005            &spec.module_id,
18006            spec.protocol,
18007            // The child exits on its own; no signal may change the exit this
18008            // test classifies.
18009            StopNotice::SentOverConnection,
18010            &registry,
18011            None,
18012            &snapshot,
18013            &runtime.terminal_ring,
18014            &runtime.spawn_events,
18015            child,
18016            Duration::from_secs(1),
18017            ModuleState::Stopped,
18018            Some(false),
18019            &super::FailureContext::operator(),
18020        )
18021        .await
18022        .unwrap();
18023
18024        let state = lock_snapshot(&snapshot).unwrap();
18025        assert_eq!(
18026            state.last_exit.as_ref().map(|exit| exit.kind),
18027            Some(ExitKind::Crash)
18028        );
18029        assert_eq!(state.lifetime_restarts, 0);
18030        assert_eq!(state.crash_restarts.len(), 0);
18031    }
18032
18033    #[test]
18034    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
18035        // The server's generic fatal-routing branch only knows that the
18036        // connection failed; it does not know that the daemon deliberately
18037        // initiated a process-killing severance. Keep this seam explicit so a
18038        // future connection error path cannot silently reintroduce the stale
18039        // exemption that mislabels a later genuine crash.
18040        assert!(!include_str!("server.rs")
18041            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
18042    }
18043
18044    /// The `route.closed` `drained` value must be the quiescence wait's own
18045    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
18046    /// measurement at all and `false` is the one honest constant. This is the exact
18047    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
18048    /// on every return path, including the one that used to return early via `?`
18049    /// with `route.closing` already sent and no `route.closed` ever following.
18050    #[test]
18051    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
18052        assert!(drained_after_quiescence_wait(&Ok(true)));
18053        assert!(!drained_after_quiescence_wait(&Ok(false)));
18054        assert!(!drained_after_quiescence_wait(&Err(
18055            SuperviseError::StatePoisoned { module_id: None }
18056        )));
18057    }
18058
18059    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
18060    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
18061    /// already reaped out-of-band) still leaves a terminal record rather than none
18062    /// at all. Triggering the real `wait()` I/O error from an integration test would
18063    /// need a genuine already-reaped-child race, which is OS-specific and not
18064    /// something this suite attempts elsewhere; this test instead verifies the
18065    /// record produced for that arm end-to-end through the real `TerminalRing`, and
18066    /// the call site itself is verified by inspection to sit in that exact arm.
18067    #[test]
18068    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
18069        let ring = Arc::new(Mutex::new(TerminalRing::new(
18070            TerminalRingConfig::default(),
18071            0,
18072        )));
18073        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
18074
18075        let snapshot = ring.lock().unwrap().snapshot();
18076        assert_eq!(snapshot.entries.len(), 1);
18077        let entry = &snapshot.entries[0];
18078        assert_eq!(entry.exit_code, None);
18079        assert_eq!(entry.exit_signal, None);
18080        assert_eq!(entry.disposition, TerminalDisposition::Failed);
18081    }
18082
18083    /// A launch is numbered before it starts. One that never runs keeps its
18084    /// number, and the next launch of the module is numbered after it.
18085    #[test]
18086    fn spawn_generation_of_a_failed_launch_is_never_reused() {
18087        let feed = super::SpawnEventFeed::default();
18088        feed.configure_incarnation("launch-numbering".to_string());
18089        let (incarnation, failed) = feed.allocate_generation("relaunched");
18090        assert_eq!(incarnation, "launch-numbering");
18091        let (_, next) = feed.allocate_generation("relaunched");
18092        assert!(next > failed);
18093        assert_eq!(feed.emit_spawned("relaunched", Some(next), 7, 1), next);
18094        // A process with no pre-allocated number is numbered after both.
18095        assert!(feed.emit_spawned("relaunched", None, 8, 2) > next);
18096        assert_eq!(
18097            feed.allocate_generation("other").1,
18098            1,
18099            "numbering is per module"
18100        );
18101    }
18102
18103    #[test]
18104    fn wait_error_exit_path_preserves_spawn_event_density() {
18105        let feed = super::SpawnEventFeed::default();
18106        feed.configure_incarnation("wait-error-density".to_string());
18107        feed.emit_spawned("wait-error", None, 41, 1);
18108        let ring = Arc::new(Mutex::new(TerminalRing::new(
18109            TerminalRingConfig::default(),
18110            0,
18111        )));
18112
18113        record_wait_error_terminal("wait-error", &ring, &feed);
18114        feed.emit_spawned("after-wait-error", None, 42, 2);
18115
18116        let state = feed.0.lock().unwrap();
18117        let sequences = state
18118            .events
18119            .iter()
18120            .map(|event| event.cursor.seq)
18121            .collect::<Vec<_>>();
18122        assert_eq!(sequences, vec![1, 2, 3]);
18123        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
18124        assert_eq!(state.events[1].exit_code, None);
18125        assert_eq!(state.events[1].exit_signal, None);
18126    }
18127
18128    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
18129    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
18130    /// not a clean exit it never actually observed.
18131    #[test]
18132    fn wait_error_exit_report_is_classified_as_a_crash() {
18133        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
18134    }
18135}
18136
18137#[cfg(all(test, unix))]
18138mod restart_ordering_tests {
18139    use super::*;
18140    use std::io::Write;
18141    use tokio::io::AsyncReadExt;
18142
18143    #[test]
18144    #[ignore = "child process fixture"]
18145    fn serving_child() {
18146        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
18147            return;
18148        };
18149        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
18150        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
18151        for stream in listener.incoming() {
18152            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
18153        }
18154    }
18155
18156    async fn restart_before_publication(requests: usize) {
18157        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
18158        let address_file = home.path().join("address");
18159        let spec = ModuleSpec {
18160            script: None,
18161            module_id: "restart-serving-child".into(),
18162            program: std::env::current_exe().unwrap(),
18163            args: vec![
18164                "--exact".into(),
18165                "supervise::restart_ordering_tests::serving_child".into(),
18166                "--ignored".into(),
18167                "--nocapture".into(),
18168            ],
18169            env: vec![(
18170                "RESTART_TEST_ADDRESS_FILE".into(),
18171                address_file.display().to_string(),
18172            )],
18173            reserved: false,
18174            reserved_prefixes: vec![],
18175            protocol: ModuleProtocol::None,
18176            overlap: Default::default(),
18177            carry_containment: false,
18178        };
18179        let mut supervisor =
18180            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18181        let (spawned_tx, spawned_rx) = oneshot::channel();
18182        let spawned_tx = Mutex::new(Some(spawned_tx));
18183        let (release_tx, release_rx) = std::sync::mpsc::channel();
18184        let release_rx = Mutex::new(release_rx);
18185        // Hold publication, not the child: it must answer a real request before
18186        // the restart enters the queue, even though status still has generation 0.
18187        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
18188            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
18189            release_rx
18190                .lock()
18191                .unwrap()
18192                .recv_timeout(Duration::from_secs(20))
18193                .unwrap();
18194        })));
18195        let registered = supervisor
18196            .register_configured_with_health(
18197                spec.clone(),
18198                true,
18199                HealthConfig::default(),
18200                None,
18201                RestartPolicy::new(3, Duration::ZERO),
18202                DEFAULT_REGISTER_TIMEOUT,
18203            )
18204            .unwrap();
18205        let module = registered.module.clone();
18206        let startup = tokio::task::spawn_blocking(move || {
18207            supervisor.start_registered_configured(registered, None, true)
18208        });
18209        spawned_rx.await.unwrap();
18210        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18211            let address = loop {
18212                if let Ok(address) = std::fs::read_to_string(&address_file) {
18213                    if !address.is_empty() {
18214                        break address;
18215                    }
18216                }
18217                tokio::time::sleep(Duration::from_millis(1)).await;
18218            };
18219            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
18220            let mut reply = String::new();
18221            stream.read_to_string(&mut reply).await.unwrap();
18222            reply.trim().parse::<u32>().unwrap()
18223        })
18224        .await
18225        .expect("child must serve before generation publication");
18226        assert_eq!(
18227            lock_snapshot(&module.inner.snapshot)
18228                .unwrap()
18229                .spawn_generation,
18230            0
18231        );
18232        let mut replies = Vec::new();
18233        for _ in 0..requests {
18234            let mut restart = Box::pin(module.restart(Some(0)));
18235            // Poll the public API through queue admission, leaving only its ACK
18236            // pending. The monitor cannot consume the command until released.
18237            std::future::poll_fn(|cx| {
18238                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
18239                std::task::Poll::Ready(())
18240            })
18241            .await;
18242            replies.push(restart);
18243        }
18244        release_tx.send(()).unwrap();
18245        startup.await.unwrap().unwrap();
18246        for reply in replies {
18247            reply.await.unwrap();
18248        }
18249        // Restart schedules its replacement after completing the handler. Wait
18250        // for that separate actor turn before placing the command barrier.
18251        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
18252            loop {
18253                if lock_snapshot(&module.inner.snapshot)
18254                    .unwrap()
18255                    .spawn_generation
18256                    > 1
18257                {
18258                    break;
18259                }
18260                tokio::time::sleep(Duration::from_millis(1)).await;
18261            }
18262        })
18263        .await;
18264        // Commands are handled serially. The update-spec reply proves every
18265        // restart handler has finished before we inspect the snapshot; restart's
18266        // own ACK only acknowledges initiation, not replacement completion.
18267        module.update_spec_for_test(spec).await.unwrap();
18268        let (generation, pid) = {
18269            let state = lock_snapshot(&module.inner.snapshot).unwrap();
18270            (state.spawn_generation, state.pid)
18271        };
18272        module.stop().await.unwrap();
18273        replacement.expect("restart must publish a replacement for the already-serving child");
18274        assert_ne!(
18275            pid,
18276            Some(old_pid),
18277            "a serving child predating the restart must be replaced"
18278        );
18279        assert_eq!(generation, 2, "queued restarts must share one replacement");
18280    }
18281
18282    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18283    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
18284        restart_before_publication(1).await;
18285    }
18286
18287    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18288    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
18289        restart_before_publication(2).await;
18290    }
18291
18292    #[tokio::test]
18293    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
18294        let supervisor =
18295            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18296        let registered = supervisor
18297            .register_configured_with_health(
18298                ModuleSpec {
18299                    script: None,
18300                    module_id: "queued-before-replacement".into(),
18301                    // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on
18302                    // macOS a module whose running image is not the configured
18303                    // program is refused once its exec is confirmed (see
18304                    // `record_write_stall_tests::sleeping_spec`).
18305                    program: "/bin/sleep".into(),
18306                    args: vec!["600".into()],
18307                    env: vec![],
18308                    reserved: false,
18309                    reserved_prefixes: vec![],
18310                    protocol: ModuleProtocol::None,
18311                    overlap: Default::default(),
18312                    carry_containment: false,
18313                },
18314                true,
18315                HealthConfig::default(),
18316                None,
18317                RestartPolicy::default(),
18318                DEFAULT_REGISTER_TIMEOUT,
18319            )
18320            .unwrap();
18321        let RegisteredConfiguredModule {
18322            mut spec,
18323            mut runtime,
18324            module,
18325            mut commands,
18326        } = registered;
18327        let snapshot = &module.inner.snapshot;
18328        let mut restart = Box::pin(module.restart(Some(0)));
18329        std::future::poll_fn(|cx| {
18330            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
18331            std::task::Poll::Ready(())
18332        })
18333        .await;
18334        // Drive the actor explicitly so a replacement is definitely spawned
18335        // after receipt, but before this queued command is dequeued.
18336        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
18337        let replacement_pid = child.as_ref().unwrap().id();
18338        let command = commands.recv().await.unwrap();
18339        assert!(
18340            handle_supervisor_command(
18341                command,
18342                &mut spec,
18343                &mut runtime,
18344                &supervisor.registry,
18345                &supervisor.process_liveness,
18346                snapshot,
18347                &mut child,
18348                &mut commands,
18349                &mut VecDeque::new(),
18350            )
18351            .await
18352        );
18353        restart.await.unwrap();
18354        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
18355        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
18356        if let Some(mut child) = child {
18357            child.start_kill().unwrap();
18358            child.wait().await.unwrap();
18359        }
18360        assert_eq!(
18361            kept_pid, replacement_pid,
18362            "a newer replacement must survive"
18363        );
18364        assert!(
18365            !pending,
18366            "a satisfied restart must not schedule another spawn"
18367        );
18368    }
18369}
18370
18371#[cfg(all(test, unix))]
18372mod health_event_tests {
18373    use super::*;
18374    use cortexkit_test_support::ScratchDir;
18375    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
18376
18377    struct Actor {
18378        module: SupervisedModule,
18379        registry: Arc<Registry>,
18380        forwarding: Arc<ForwardingTable>,
18381        spec: ModuleSpec,
18382        health: HealthConfig,
18383        rx: mpsc::Receiver<crate::router::OutboundFrame>,
18384        roster: ChildRoster,
18385        _home: ScratchDir,
18386    }
18387
18388    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
18389        // OS exec and socket readiness are real I/O. Stay runnable while waiting
18390        // for an observed condition, rather than letting paused time auto-advance
18391        // deadlines or assuming a fixed number of yields finishes the work.
18392        let deadline = std::time::Instant::now() + Duration::from_secs(10);
18393        let tick = Instant::now();
18394        loop {
18395            if let Some(value) = observe() {
18396                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
18397                return value;
18398            }
18399            assert!(
18400                std::time::Instant::now() < deadline,
18401                "timed out waiting for {reason}"
18402            );
18403            tokio::task::yield_now().await;
18404        }
18405    }
18406
18407    impl Actor {
18408        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
18409            Self::start_with_policy(
18410                protocol,
18411                cadence,
18412                DEFAULT_REGISTER_TIMEOUT,
18413                RestartPolicy::new(3, Duration::ZERO),
18414            )
18415            .await
18416        }
18417
18418        async fn start_with_policy(
18419            protocol: ModuleProtocol,
18420            cadence: Duration,
18421            register_timeout: Duration,
18422            policy: RestartPolicy,
18423        ) -> Self {
18424            let home = ScratchDir::new("health-events");
18425            let registry = Arc::new(Registry::default());
18426            let forwarding = Arc::new(ForwardingTable::default());
18427            let health = HealthConfig {
18428                cadence,
18429                deadline: Duration::from_secs(3600),
18430                ..HealthConfig::default()
18431            };
18432            let supervisor = Supervisor::new_for_test(registry.clone(), policy)
18433                .with_forwarding(forwarding.clone())
18434                .with_health_config(health.clone());
18435            let spec = ModuleSpec {
18436                script: None,
18437                module_id: "event-health-child".into(),
18438                // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on macOS
18439                // the privacy trampoline refuses a module whose running image
18440                // is not the configured program, and `start` below waits for
18441                // the process to be confirmed (`reported_pid`). The shell form
18442                // passed only while the image check ran before the shell had
18443                // replaced itself with `sleep` (see
18444                // `record_write_stall_tests::sleeping_spec`).
18445                program: PathBuf::from("/bin/sleep"),
18446                // The process stays alive but has no real bus peer. Registrations
18447                // below exercise the same registry writes as accepted HELLOs.
18448                args: vec!["600".into()],
18449                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18450                    .into_iter()
18451                    .map(|key| (key.into(), home.path().display().to_string()))
18452                    .collect(),
18453                reserved: false,
18454                reserved_prefixes: vec![],
18455                protocol,
18456                overlap: Default::default(),
18457                carry_containment: false,
18458            };
18459            let module = supervisor
18460                .supervise_configured_requested(
18461                    spec.clone(),
18462                    true,
18463                    health.clone(),
18464                    Some(0),
18465                    policy,
18466                    register_timeout,
18467                    None,
18468                )
18469                .unwrap();
18470            let (_, rx) = mpsc::channel(8);
18471            let actor = Self {
18472                module,
18473                registry,
18474                forwarding,
18475                spec,
18476                health,
18477                rx,
18478                roster: supervisor.child_roster.clone(),
18479                _home: home,
18480            };
18481            actor
18482                .wait_for_select(|checkpoint| checkpoint.generation > 0)
18483                .await;
18484            actor
18485        }
18486
18487        async fn expire_registration_wait(&self, remaining: Duration) {
18488            tokio::time::advance(remaining).await;
18489            wait_for("registration timeout TERM grace", || {
18490                lock_snapshot(&self.module.inner.snapshot)
18491                    .unwrap()
18492                    .actor_select
18493                    .is_none()
18494                    .then_some(())
18495            })
18496            .await;
18497            tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
18498        }
18499
18500        fn turns(&self) -> u64 {
18501            lock_snapshot(&self.module.inner.snapshot)
18502                .unwrap()
18503                .actor_turns
18504        }
18505
18506        async fn wait_for_select(
18507            &self,
18508            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
18509        ) -> ActorSelectCheckpoint {
18510            wait_for("actor select checkpoint", || {
18511                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
18512                state.actor_select.filter(|checkpoint| {
18513                    checkpoint.generation == state.spawn_generation
18514                        && state.state == ModuleState::Running
18515                        && state.process_alive
18516                        && state.reported_pid().is_some()
18517                        && expected(checkpoint)
18518                })
18519            })
18520            .await
18521        }
18522
18523        fn hello(&mut self, connection: u64, health: bool) {
18524            let manifest =
18525                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
18526                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
18527                    .build();
18528            let connection = ConnectionId::new(connection);
18529            let (tx, rx) = mpsc::channel(8);
18530            self.rx = rx;
18531            self.forwarding
18532                .register_module_connection(
18533                    connection,
18534                    self.spec.module_id.clone(),
18535                    subc_protocol::PROTOCOL_VERSION,
18536                    Concurrency::ModuleManaged,
18537                    FrameSink::new(tx),
18538                )
18539                .unwrap();
18540            self.registry
18541                .register_with_control_ops(
18542                    manifest,
18543                    subc_protocol::PROTOCOL_VERSION,
18544                    connection,
18545                    if health {
18546                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
18547                    } else {
18548                        vec![]
18549                    },
18550                )
18551                .unwrap();
18552        }
18553
18554        async fn probe(&mut self) -> crate::router::OutboundFrame {
18555            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
18556                Ok(frame) => Some(frame),
18557                Err(mpsc::error::TryRecvError::Empty) => None,
18558                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
18559            })
18560            .await;
18561            let body: Value = serde_json::from_slice(&frame.body).unwrap();
18562            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
18563            frame
18564        }
18565
18566        fn no_probe(&mut self) {
18567            assert!(matches!(
18568                self.rx.try_recv(),
18569                Err(mpsc::error::TryRecvError::Empty)
18570            ));
18571        }
18572
18573        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
18574            self.forwarding
18575                .complete_module_control_rpc(
18576                    ConnectionId::new(connection),
18577                    frame.header.corr,
18578                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
18579                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
18580                        status: HealthStatus::Ok,
18581                        detail: None,
18582                        metrics: None,
18583                    }),
18584                )
18585                .unwrap();
18586        }
18587    }
18588
18589    #[tokio::test(start_paused = true)]
18590    async fn registration_timeout_kills_at_120_seconds_and_respawns() {
18591        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18592        let initial = actor.module.status().unwrap();
18593        tokio::time::advance(Duration::from_secs(119)).await;
18594        tokio::task::yield_now().await;
18595        assert_eq!(actor.module.status().unwrap().pid, initial.pid);
18596        assert!(actor.module.terminal_history().entries.is_empty());
18597        actor.expire_registration_wait(Duration::from_secs(1)).await;
18598        actor
18599            .wait_for_select(|at| at.generation > initial.spawn_generation)
18600            .await;
18601        let status = actor.module.status().unwrap();
18602        assert_ne!(status.pid, initial.pid);
18603        assert!(!cortexkit_test_support::process_alive(
18604            initial.pid.unwrap() as i32
18605        ));
18606        assert_eq!(status.restart_count, 1);
18607        assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
18608        let entries = actor.module.terminal_history().entries;
18609        assert_eq!(entries.len(), 1);
18610        assert_eq!(
18611            entries[0].disposition_detail.as_deref(),
18612            Some("registration_timeout")
18613        );
18614        actor.module.stop().await.unwrap();
18615    }
18616
18617    #[tokio::test(start_paused = true)]
18618    async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
18619        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18620        let pid = actor.module.status().unwrap().pid;
18621        tokio::time::advance(Duration::from_secs(119)).await;
18622        actor.hello(1, false);
18623        let registered = actor
18624            .wait_for_select(|at| at.registered_connection.is_some())
18625            .await;
18626        assert_eq!(
18627            registered.wake_after, None,
18628            "HELLO must cancel the deadline"
18629        );
18630        tokio::time::advance(Duration::from_secs(300)).await;
18631        tokio::task::yield_now().await;
18632        assert_eq!(actor.module.status().unwrap().pid, pid);
18633        assert!(actor.module.terminal_history().entries.is_empty());
18634        // A disconnect after the first HELLO must not re-arm a startup timer.
18635        actor
18636            .registry
18637            .deregister_connection(ConnectionId::new(1))
18638            .unwrap();
18639        actor
18640            .wait_for_select(|at| at.registered_connection.is_none())
18641            .await;
18642        tokio::time::advance(Duration::from_secs(300)).await;
18643        tokio::task::yield_now().await;
18644        assert_eq!(actor.module.status().unwrap().pid, pid);
18645        assert!(actor.module.terminal_history().entries.is_empty());
18646        actor.module.stop().await.unwrap();
18647
18648        let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18649        let pid = transient.module.status().unwrap().pid;
18650        tokio::time::advance(Duration::from_secs(119)).await;
18651        let turns = transient.turns();
18652        // Both writes happen before the supervisor can refresh the registry.
18653        // An accepted HELLO must remain evidence after its connection is gone.
18654        transient.hello(1, false);
18655        transient
18656            .registry
18657            .deregister_connection(ConnectionId::new(1))
18658            .unwrap();
18659        let parked = transient.wait_for_select(|at| at.turn > turns).await;
18660        assert_eq!(
18661            parked.wake_after, None,
18662            "transient HELLO must cancel the deadline"
18663        );
18664        tokio::time::advance(Duration::from_secs(300)).await;
18665        tokio::task::yield_now().await;
18666        assert_eq!(transient.module.status().unwrap().pid, pid);
18667        assert!(transient.module.terminal_history().entries.is_empty());
18668        transient.module.stop().await.unwrap();
18669    }
18670
18671    #[tokio::test(start_paused = true)]
18672    async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
18673        let actor = Actor::start_with_policy(
18674            ModuleProtocol::Subc,
18675            Duration::from_secs(30),
18676            Duration::from_secs(10),
18677            RestartPolicy::new(1, Duration::ZERO),
18678        )
18679        .await;
18680        let generation = actor.module.status().unwrap().spawn_generation;
18681        actor
18682            .expire_registration_wait(Duration::from_secs(10))
18683            .await;
18684        actor.wait_for_select(|at| at.generation > generation).await;
18685        actor
18686            .expire_registration_wait(Duration::from_secs(10))
18687            .await;
18688        let retry = wait_for("failed retry schedule", || {
18689            let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
18690            (state.state == ModuleState::Failed)
18691                .then_some(state.failed_retry.pending)
18692                .flatten()
18693        })
18694        .await;
18695        assert_eq!(retry.delay, Duration::from_secs(600));
18696        assert_eq!(retry.attempt, 1);
18697        let entries = actor.module.terminal_history().entries;
18698        assert_eq!(entries.len(), 2);
18699        assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
18700        assert!(entries[1]
18701            .disposition_detail
18702            .as_deref()
18703            .unwrap()
18704            .contains("registration_timeout"));
18705        tokio::time::advance(Duration::from_secs(599)).await;
18706        tokio::task::yield_now().await;
18707        assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
18708        tokio::time::advance(Duration::from_secs(1)).await;
18709        actor
18710            .wait_for_select(|at| at.generation > generation + 1)
18711            .await;
18712        // The slow retry is another ordinary spawn, with its own HELLO budget.
18713        actor
18714            .expire_registration_wait(Duration::from_secs(10))
18715            .await;
18716        wait_for("slow retry registration timeout", || {
18717            (actor.module.terminal_history().entries.len() == 3).then_some(())
18718        })
18719        .await;
18720        actor.module.stop().await.unwrap();
18721    }
18722
18723    #[tokio::test(start_paused = true)]
18724    async fn registration_timeout_honours_ten_second_override() {
18725        let actor = Actor::start_with_policy(
18726            ModuleProtocol::Subc,
18727            Duration::from_secs(30),
18728            Duration::from_secs(10),
18729            RestartPolicy::new(3, Duration::ZERO),
18730        )
18731        .await;
18732        let generation = actor.module.status().unwrap().spawn_generation;
18733        tokio::time::advance(Duration::from_secs(9)).await;
18734        tokio::task::yield_now().await;
18735        assert!(actor.module.terminal_history().entries.is_empty());
18736        actor.expire_registration_wait(Duration::from_secs(1)).await;
18737        actor.wait_for_select(|at| at.generation > generation).await;
18738        assert_eq!(actor.module.terminal_history().entries.len(), 1);
18739        actor.module.stop().await.unwrap();
18740    }
18741
18742    #[tokio::test]
18743    async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
18744        use std::os::unix::process::{CommandExt, ExitStatusExt};
18745
18746        struct Helper(std::process::Child);
18747        impl Drop for Helper {
18748            fn drop(&mut self) {
18749                // This handle still owns the helper until wait returns. Never
18750                // clean up its old process group after the leader is reaped.
18751                if matches!(self.0.try_wait(), Ok(None)) {
18752                    let _ = self.0.kill();
18753                    let _ = self.0.wait();
18754                }
18755            }
18756        }
18757
18758        let home = ScratchDir::new("registration-timeout-helper");
18759        let supervisor = Supervisor::new_for_test(
18760            Arc::new(Registry::default()),
18761            RestartPolicy::new(0, Duration::ZERO),
18762        );
18763        let spec = ModuleSpec {
18764            script: None,
18765            module_id: "registration-timeout-group".into(),
18766            program: "/bin/sleep".into(),
18767            args: vec!["120".into()],
18768            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18769                .into_iter()
18770                .map(|key| (key.into(), home.path().display().to_string()))
18771                .collect(),
18772            reserved: false,
18773            reserved_prefixes: vec![],
18774            protocol: ModuleProtocol::Subc,
18775            overlap: Default::default(),
18776            carry_containment: false,
18777        };
18778        let module = supervisor
18779            .supervise_configured_requested(
18780                spec,
18781                true,
18782                HealthConfig::default(),
18783                None,
18784                RestartPolicy::new(0, Duration::ZERO),
18785                Duration::from_secs(10),
18786                None,
18787            )
18788            .unwrap();
18789        let leader = timeout(Duration::from_secs(5), async {
18790            loop {
18791                if let Some(pid) = module.status().unwrap().pid {
18792                    break pid;
18793                }
18794                sleep(Duration::from_millis(5)).await;
18795            }
18796        })
18797        .await
18798        .expect("supervised group leader must be reported before the deadline");
18799        assert!(leader > 1);
18800        let ready = home.join("helper.ready");
18801        // Own the helper separately so even a broken group kill can be cleaned
18802        // up by its own unreaped child handle, without signalling a stale pgid.
18803        let mut helper = Helper(
18804            std::process::Command::new("/bin/sh")
18805                .args([
18806                    "-c",
18807                    "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
18808                    "helper",
18809                ])
18810                .arg(&ready)
18811                .process_group(i32::try_from(leader).unwrap())
18812                .stdout(std::process::Stdio::null())
18813                .stderr(std::process::Stdio::null())
18814                .spawn()
18815                .unwrap(),
18816        );
18817        timeout(Duration::from_secs(5), async {
18818            while !ready.exists() {
18819                assert!(helper.0.try_wait().unwrap().is_none());
18820                sleep(Duration::from_millis(5)).await;
18821            }
18822        })
18823        .await
18824        .expect("helper must install TERM ignore before the registration deadline");
18825        let helper_pid =
18826            rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
18827        rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
18828        sleep(Duration::from_millis(20)).await;
18829        assert!(
18830            helper.0.try_wait().unwrap().is_none(),
18831            "helper must ignore TERM"
18832        );
18833
18834        timeout(Duration::from_secs(20), async {
18835            while module.state().unwrap() != ModuleState::Failed {
18836                sleep(Duration::from_millis(5)).await;
18837            }
18838        })
18839        .await
18840        .expect("registration timeout must stop and reap the group leader");
18841        let helper_exit = timeout(Duration::from_secs(3), async {
18842            loop {
18843                if let Some(status) = helper.0.try_wait().unwrap() {
18844                    break status;
18845                }
18846                sleep(Duration::from_millis(5)).await;
18847            }
18848        })
18849        .await
18850        .expect("TERM-ignoring helper survived the registration timeout's group KILL");
18851        assert_eq!(helper_exit.signal(), Some(9));
18852        assert!(!cortexkit_test_support::process_alive(
18853            i32::try_from(leader).unwrap()
18854        ));
18855        assert_eq!(
18856            module.status().unwrap().last_exit.unwrap().kind,
18857            ExitKind::Crash
18858        );
18859        assert!(module.terminal_history().entries[0]
18860            .disposition_detail
18861            .as_deref()
18862            .unwrap()
18863            .contains("registration_timeout"));
18864        module.stop().await.unwrap();
18865    }
18866
18867    #[tokio::test(start_paused = true)]
18868    async fn registration_timeout_does_not_count_an_earlier_exit() {
18869        let actor = Actor::start_with_policy(
18870            ModuleProtocol::Subc,
18871            Duration::from_secs(30),
18872            DEFAULT_REGISTER_TIMEOUT,
18873            RestartPolicy::new(0, Duration::ZERO),
18874        )
18875        .await;
18876        let pid = actor.module.status().unwrap().pid.unwrap();
18877        rustix::process::kill_process(
18878            rustix::process::Pid::from_raw(pid as i32).unwrap(),
18879            rustix::process::Signal::TERM,
18880        )
18881        .unwrap();
18882        wait_for("early exit", || {
18883            (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
18884        })
18885        .await;
18886        tokio::time::advance(Duration::from_secs(121)).await;
18887        tokio::task::yield_now().await;
18888        let entries = actor.module.terminal_history().entries;
18889        assert_eq!(entries.len(), 1);
18890        assert!(!entries[0]
18891            .disposition_detail
18892            .as_deref()
18893            .unwrap()
18894            .contains("registration_timeout"));
18895        assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
18896        actor.module.stop().await.unwrap();
18897    }
18898
18899    #[tokio::test(start_paused = true)]
18900    async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
18901        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18902        let pid = actor.module.status().unwrap().pid;
18903        tokio::time::advance(Duration::from_secs(119)).await;
18904        actor.roster.close();
18905        let turns = actor.turns();
18906        tokio::time::advance(Duration::from_secs(300)).await;
18907        let parked = actor.wait_for_select(|at| at.turn > turns).await;
18908        assert_eq!(parked.wake_after, None);
18909        assert_eq!(actor.module.status().unwrap().pid, pid);
18910        assert!(actor.module.terminal_history().entries.is_empty());
18911        actor.module.stop().await.unwrap();
18912        let status = actor.module.status().unwrap();
18913        assert_eq!(status.restart_count, 0);
18914        assert_eq!(
18915            actor.module.terminal_history().entries[0].disposition,
18916            TerminalDisposition::DaemonShutdown
18917        );
18918    }
18919
18920    #[tokio::test(start_paused = true)]
18921    async fn no_health_children_park_for_sixty_seconds() {
18922        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
18923        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18924        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18925        unadvertised.hello(1, false);
18926        // Include any one-off registration wake in the budget; receiving a
18927        // no-health HELLO must not turn a parked child into a polling child.
18928        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
18929        // a single 60 s jump would only observe one expired timer.
18930        for _ in 0..6000 {
18931            tokio::time::advance(Duration::from_millis(10)).await;
18932            tokio::task::yield_now().await;
18933        }
18934        for actor in [&none, &unregistered, &unadvertised] {
18935            assert!(
18936                actor.turns() <= 5,
18937                "no-health actor woke {} times",
18938                actor.turns()
18939            );
18940            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
18941        }
18942    }
18943
18944    #[tokio::test(start_paused = true)]
18945    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
18946        // With a zero cadence the first probe is due at once, so this measures
18947        // only how fast the registration notification starts health probing,
18948        // not the normal 30-33 s wait before a first probe (checked below).
18949        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
18950        let tick = Instant::now();
18951        actor.hello(1, true);
18952        actor.probe().await;
18953        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
18954    }
18955
18956    #[tokio::test(start_paused = true)]
18957    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
18958        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18959        let hello_tick = Instant::now();
18960        actor.hello(1, true);
18961        let armed = actor
18962            .wait_for_select(|checkpoint| {
18963                checkpoint.registered_connection == Some(ConnectionId::new(1))
18964                    && checkpoint.next_probe_at.is_some()
18965            })
18966            .await;
18967        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18968        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
18969        let first_deadline = hello_tick + delay;
18970        assert_eq!(armed.next_probe_at, Some(first_deadline));
18971        tokio::time::advance(Duration::from_secs(29)).await;
18972        let turns = actor.turns();
18973        actor
18974            .registry
18975            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
18976            .unwrap();
18977        let updated = actor
18978            .wait_for_select(|checkpoint| checkpoint.turn > turns)
18979            .await;
18980        assert_eq!(
18981            updated.next_probe_at,
18982            Some(first_deadline),
18983            "catalog update must not reset cadence"
18984        );
18985        actor.no_probe();
18986        tokio::time::advance(first_deadline - Instant::now()).await;
18987        let frame = actor.probe().await;
18988        actor.answer(1, frame);
18989        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
18990        let next_deadline = Instant::now() + next;
18991        let rearmed = actor
18992            .wait_for_select(|checkpoint| {
18993                checkpoint
18994                    .next_probe_at
18995                    .is_some_and(|deadline| deadline > first_deadline)
18996            })
18997            .await;
18998        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
18999        tokio::time::advance(next - Duration::from_nanos(1)).await;
19000        actor.no_probe();
19001        tokio::time::advance(Duration::from_nanos(1)).await;
19002        actor.probe().await;
19003    }
19004
19005    #[tokio::test(start_paused = true)]
19006    async fn disconnect_disarms_and_reconnect_rearms_health() {
19007        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
19008        actor.hello(1, true);
19009        actor
19010            .wait_for_select(|checkpoint| {
19011                checkpoint.registered_connection == Some(ConnectionId::new(1))
19012                    && checkpoint.next_probe_at.is_some()
19013            })
19014            .await;
19015        let turns = actor.turns();
19016        let tick = Instant::now();
19017        actor
19018            .registry
19019            .deregister_connection(ConnectionId::new(1))
19020            .unwrap();
19021        let disarmed = actor
19022            .wait_for_select(|checkpoint| {
19023                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
19024            })
19025            .await;
19026        assert_eq!(disarmed.next_probe_at, None);
19027        assert_eq!(disarmed.wake_after, None);
19028        assert_eq!(
19029            Instant::now(),
19030            tick,
19031            "disconnect must disarm in the notification tick"
19032        );
19033        tokio::time::advance(Duration::from_secs(60)).await;
19034        actor.no_probe();
19035        let reconnect_tick = Instant::now();
19036        actor.hello(2, true);
19037        let rearmed = actor
19038            .wait_for_select(|checkpoint| {
19039                checkpoint.registered_connection == Some(ConnectionId::new(2))
19040                    && checkpoint.next_probe_at.is_some()
19041            })
19042            .await;
19043        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
19044        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
19045        tokio::time::advance(delay).await;
19046        actor.probe().await;
19047    }
19048
19049    #[tokio::test(start_paused = true)]
19050    async fn reregistration_without_health_disarms_the_old_schedule() {
19051        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
19052        actor.hello(1, true);
19053        actor
19054            .wait_for_select(|checkpoint| {
19055                checkpoint.registered_connection == Some(ConnectionId::new(1))
19056                    && checkpoint.next_probe_at.is_some()
19057            })
19058            .await;
19059        actor
19060            .registry
19061            .deregister_connection(ConnectionId::new(1))
19062            .unwrap();
19063        actor.hello(2, false);
19064        let disarmed = actor
19065            .wait_for_select(|checkpoint| {
19066                checkpoint.registered_connection == Some(ConnectionId::new(2))
19067            })
19068            .await;
19069        assert_eq!(disarmed.next_probe_at, None);
19070        assert_eq!(disarmed.wake_after, None);
19071        tokio::time::advance(Duration::from_secs(60)).await;
19072        actor.no_probe();
19073    }
19074
19075    #[tokio::test(start_paused = true)]
19076    async fn restart_waits_for_its_own_hello_before_rearming_health() {
19077        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
19078        actor.hello(1, true);
19079        let armed = actor
19080            .wait_for_select(|checkpoint| {
19081                checkpoint.registered_connection == Some(ConnectionId::new(1))
19082                    && checkpoint.next_probe_at.is_some()
19083            })
19084            .await;
19085        // Simulate the old process's connection closing before the restart
19086        // tears the process down. The replacement process has started but has
19087        // not sent its HELLO, so it is not registered.
19088        actor
19089            .registry
19090            .deregister_connection(ConnectionId::new(1))
19091            .unwrap();
19092        actor.module.restart(Some(0)).await.unwrap();
19093        let parked = actor
19094            .wait_for_select(|checkpoint| {
19095                checkpoint.generation > armed.generation
19096                    && checkpoint.registered_connection.is_none()
19097                    && checkpoint.next_probe_at.is_none()
19098            })
19099            .await;
19100        assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
19101        actor.rx.close();
19102        // Drain the stop notice the restart queued for the old process's connection,
19103        // so it isn't mistaken for traffic to the replacement.
19104        while actor.rx.try_recv().is_ok() {}
19105        let turns = actor.turns();
19106        tokio::time::advance(Duration::from_secs(60)).await;
19107        let after = actor.turns();
19108        eprintln!(
19109            "restart park turns: before={turns} after={after} delta={}",
19110            after - turns
19111        );
19112        assert_eq!(after, turns, "replacement must park before HELLO");
19113        let hello_tick = Instant::now();
19114        actor.hello(2, true);
19115        let rearmed = actor
19116            .wait_for_select(|checkpoint| {
19117                checkpoint.registered_connection == Some(ConnectionId::new(2))
19118                    && checkpoint.next_probe_at.is_some()
19119            })
19120            .await;
19121        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
19122        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
19123        tokio::time::advance(delay).await;
19124        actor.probe().await;
19125    }
19126
19127    #[tokio::test(start_paused = true)]
19128    async fn http_probe_waits_a_full_cadence_for_each_spawn() {
19129        use tokio::{
19130            io::{AsyncReadExt, AsyncWriteExt},
19131            net::TcpListener,
19132        };
19133        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
19134        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
19135        actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
19136        let (started_tx, mut started_rx) = mpsc::channel(8);
19137        let peer = tokio::spawn(async move {
19138            loop {
19139                let (mut socket, _) = listener.accept().await.unwrap();
19140                let mut request = [0; 1024];
19141                assert!(socket.read(&mut request).await.unwrap() > 0);
19142                started_tx.send(Instant::now()).await.unwrap();
19143                socket
19144                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
19145                    .await
19146                    .unwrap();
19147            }
19148        });
19149        actor
19150            .module
19151            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19152            .await
19153            .unwrap();
19154        let first_tick = Instant::now();
19155        let first = actor
19156            .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
19157            .await;
19158        assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
19159        assert!(started_rx.try_recv().is_err());
19160
19161        // Restart just before the old deadline: reusing it would give the new
19162        // process only one second to start, rather than a full cadence.
19163        tokio::time::advance(
19164            first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
19165        )
19166        .await;
19167        actor.module.restart(Some(0)).await.unwrap();
19168        let spawn_tick = Instant::now();
19169        let replacement = actor
19170            .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
19171            .await;
19172        let deadline = replacement.next_probe_at.unwrap();
19173        assert!(
19174            deadline >= spawn_tick + actor.health.cadence,
19175            "a replacement must not inherit its predecessor's probe deadline"
19176        );
19177        tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
19178        assert!(
19179            started_rx.try_recv().is_err(),
19180            "no HTTP probe during startup grace"
19181        );
19182        tokio::time::advance(deadline - Instant::now()).await;
19183        let landed = wait_for("first replacement HTTP probe", || {
19184            started_rx.try_recv().ok()
19185        })
19186        .await;
19187        assert!(landed >= spawn_tick + actor.health.cadence);
19188        wait_for("healthy replacement", || {
19189            (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
19190                .then_some(())
19191        })
19192        .await;
19193        peer.abort();
19194    }
19195
19196    #[tokio::test(start_paused = true)]
19197    async fn rescan_adds_and_removes_http_health_without_polling() {
19198        use tokio::{
19199            io::{AsyncReadExt, AsyncWriteExt},
19200            net::TcpListener,
19201        };
19202        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
19203        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
19204        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
19205        // One millisecond separates successive probes without a polling delay.
19206        actor.health.cadence = Duration::from_millis(1);
19207        let (started_tx, mut started_rx) = mpsc::channel(8);
19208        let peer = tokio::spawn(async move {
19209            loop {
19210                let (mut socket, _) = listener.accept().await.unwrap();
19211                let mut request = [0; 1024];
19212                assert!(socket.read(&mut request).await.unwrap() > 0);
19213                started_tx.send(()).await.unwrap();
19214                socket
19215                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
19216                    .await
19217                    .unwrap();
19218            }
19219        });
19220        let tick = Instant::now();
19221        let turns = actor.turns();
19222        actor
19223            .module
19224            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19225            .await
19226            .unwrap();
19227        let armed = actor
19228            .wait_for_select(|checkpoint| {
19229                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
19230            })
19231            .await;
19232        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
19233        assert_eq!(Instant::now(), tick);
19234        tokio::time::advance(Duration::from_millis(1)).await;
19235        let mut started = false;
19236        wait_for("successful HTTP probe", || {
19237            started |= started_rx.try_recv().is_ok();
19238            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
19239                .then_some(())
19240        })
19241        .await;
19242        assert!(started, "added HTTP check must start at its first deadline");
19243        assert_eq!(
19244            actor.module.status().unwrap().health.status,
19245            SupervisorHealthStatus::Ok
19246        );
19247        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
19248        actor.health.http = None;
19249        let turns = actor.turns();
19250        actor
19251            .module
19252            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19253            .await
19254            .unwrap();
19255        let parked = actor
19256            .wait_for_select(|checkpoint| {
19257                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
19258            })
19259            .await;
19260        assert_eq!(parked.wake_after, None);
19261        let turns = actor.turns();
19262        tokio::time::advance(Duration::from_secs(60)).await;
19263        assert_eq!(
19264            actor.turns(),
19265            turns,
19266            "removed HTTP check must park the actor"
19267        );
19268        assert!(
19269            started_rx.try_recv().is_err(),
19270            "removed HTTP check must stay disarmed"
19271        );
19272        assert_eq!(
19273            actor.module.status().unwrap().health,
19274            ModuleHealthStatus::default()
19275        );
19276        peer.abort();
19277    }
19278}
19279
19280#[cfg(test)]
19281mod drain_busy_reprobe_tests {
19282    //! The drain's busy-gauge wait, driven against an in-process module
19283    //! connection whose health answers the test scripts. Time is paused, so
19284    //! every elapsed duration below is exact virtual time.
19285    use super::*;
19286    use std::collections::BTreeMap;
19287    use std::sync::Mutex as StdMutex;
19288    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
19289
19290    const MODULE_ID: &str = "drain-reprobe-module";
19291    const CONNECTION: u64 = 7;
19292    const GAUGE: &str = "runs_in_flight";
19293    /// The production default for both the health cadence and the drain
19294    /// ceiling, which is the pairing that used to cost a full ceiling.
19295    const CADENCE: Duration = Duration::from_secs(30);
19296    const CEILING: Duration = Duration::from_secs(30);
19297
19298    /// How the scripted module answers each `health.check`.
19299    #[derive(Clone, Copy)]
19300    enum Answer {
19301        /// Report the gauge busy until this instant, quiet from then on, after
19302        /// taking `latency` to answer.
19303        BusyUntil { until: Instant, latency: Duration },
19304        /// Answer at once, with metrics that leave the declared gauge out.
19305        OmitsGauge,
19306        /// Never answer.
19307        Silent,
19308    }
19309
19310    struct Drain {
19311        forwarding: Arc<ForwardingTable>,
19312        runtime: SupervisorRuntimeConfig,
19313        endpoint: crate::ModuleEndpointId,
19314        /// When each probe reached the module.
19315        probes: Arc<StdMutex<Vec<Instant>>>,
19316        responder: tokio::task::JoinHandle<()>,
19317    }
19318
19319    impl Drain {
19320        fn start(answer: Answer) -> Self {
19321            let registry = Arc::new(Registry::default());
19322            let forwarding = Arc::new(ForwardingTable::default());
19323            let health = HealthConfig {
19324                cadence: CADENCE,
19325                ..HealthConfig::default()
19326            };
19327            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
19328                .with_forwarding(forwarding.clone())
19329                .with_health_config(health)
19330                .runtime_config();
19331            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
19332            let endpoint = forwarding
19333                .register_module_connection(
19334                    ConnectionId::new(CONNECTION),
19335                    MODULE_ID.to_string(),
19336                    subc_protocol::PROTOCOL_VERSION,
19337                    Concurrency::ModuleManaged,
19338                    FrameSink::new(tx),
19339                )
19340                .unwrap();
19341            let probes = Arc::new(StdMutex::new(Vec::new()));
19342            let responder = tokio::spawn({
19343                let forwarding = forwarding.clone();
19344                let probes = probes.clone();
19345                async move {
19346                    while let Some(frame) = rx.recv().await {
19347                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
19348                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
19349                        probes.lock().unwrap().push(Instant::now());
19350                        let metrics = match answer {
19351                            Answer::BusyUntil { until, latency } => {
19352                                sleep(latency).await;
19353                                let busy = u64::from(Instant::now() < until);
19354                                serde_json::json!({ GAUGE: busy })
19355                            }
19356                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
19357                            Answer::Silent => continue,
19358                        };
19359                        // A probe abandoned at its deadline has no RPC left to
19360                        // complete, so a late answer may be refused.
19361                        let _ = forwarding.complete_module_control_rpc(
19362                            ConnectionId::new(CONNECTION),
19363                            frame.header.corr,
19364                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
19365                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19366                                status: HealthStatus::Ok,
19367                                detail: None,
19368                                metrics: Some(metrics),
19369                            }),
19370                        );
19371                    }
19372                }
19373            });
19374            Self {
19375                forwarding,
19376                runtime,
19377                endpoint,
19378                probes,
19379                responder,
19380            }
19381        }
19382
19383        /// Run one by-id drain wait with `ceiling`, returning its verdict and
19384        /// how long it took.
19385        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
19386            let (drained, elapsed, _) = self.run_traced(ceiling).await;
19387            (drained, elapsed)
19388        }
19389
19390        /// [`Self::run`], also returning what the drain's probes read.
19391        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
19392            let started = Instant::now();
19393            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
19394            let drained = wait_for_forwarding_quiescence(
19395                &self.forwarding,
19396                MODULE_ID,
19397                &self.runtime,
19398                self.endpoint,
19399                started + ceiling,
19400                &[GAUGE.to_string()],
19401                DrainScope::Active,
19402                &mut trace,
19403            )
19404            .await
19405            .unwrap();
19406            (drained, started.elapsed(), trace)
19407        }
19408
19409        fn probes(&self) -> Vec<Instant> {
19410            self.probes.lock().unwrap().clone()
19411        }
19412    }
19413
19414    impl Drop for Drain {
19415        fn drop(&mut self) {
19416            self.responder.abort();
19417        }
19418    }
19419
19420    #[tokio::test(start_paused = true)]
19421    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
19422        let drain = Drain::start(Answer::BusyUntil {
19423            until: Instant::now() + Duration::from_secs(1),
19424            latency: Duration::ZERO,
19425        });
19426        let (drained, elapsed) = drain.run(CEILING).await;
19427        assert!(
19428            drained,
19429            "gauges that read zero during the drain must let it finish drained, \
19430             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
19431        );
19432        assert!(
19433            elapsed < Duration::from_secs(3),
19434            "drain took {elapsed:?} after the gauges went quiet at 1s"
19435        );
19436        let probes = drain.probes();
19437        assert!(
19438            probes.len() >= 2,
19439            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
19440        );
19441    }
19442
19443    /// The same shape in real time on a multi-threaded runtime: the first
19444    /// probe is answered busy at once, within a millisecond of the drain
19445    /// starting, and later probes read idle. The drain must keep probing
19446    /// after that first instant answer and end soon after the gauge falls.
19447    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
19448    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
19449        let drain = Drain::start(Answer::BusyUntil {
19450            until: Instant::now() + Duration::from_millis(600),
19451            latency: Duration::ZERO,
19452        });
19453        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
19454        assert!(drained, "the gauge fell at 600ms: {trace}");
19455        assert!(
19456            elapsed < Duration::from_secs(3),
19457            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
19458        );
19459        let probes = drain.probes();
19460        assert!(
19461            probes.len() >= 2 && trace.probes as usize == probes.len(),
19462            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
19463        );
19464    }
19465
19466    #[tokio::test(start_paused = true)]
19467    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
19468        let started = Instant::now();
19469        let drain = Drain::start(Answer::BusyUntil {
19470            until: started + Duration::from_secs(3600),
19471            latency: Duration::ZERO,
19472        });
19473        let (drained, elapsed) = drain.run(CEILING).await;
19474        assert!(!drained, "a module busy throughout must not drain");
19475        assert!(
19476            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
19477            "a busy module is held until the ceiling and no longer: {elapsed:?}"
19478        );
19479        // Re-probing is paced by the interval, not a busy loop: at most one
19480        // probe per interval across the ceiling, plus the first at time zero.
19481        let probes = drain.probes();
19482        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
19483        assert!(
19484            probes.len() as u128 <= max_probes,
19485            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
19486            probes.len()
19487        );
19488        for pair in probes.windows(2) {
19489            assert!(
19490                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
19491                "consecutive probes {:?} apart",
19492                pair[1] - pair[0]
19493            );
19494        }
19495    }
19496
19497    #[tokio::test(start_paused = true)]
19498    async fn a_slow_probe_is_never_overlapped_by_the_next() {
19499        // Each answer takes longer than the re-probe interval; the next probe
19500        // must wait for the previous answer and then the interval.
19501        let latency = Duration::from_secs(1);
19502        let drain = Drain::start(Answer::BusyUntil {
19503            until: Instant::now() + Duration::from_secs(3600),
19504            latency,
19505        });
19506        let (drained, _) = drain.run(Duration::from_secs(5)).await;
19507        assert!(!drained);
19508        let probes = drain.probes();
19509        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
19510        for pair in probes.windows(2) {
19511            assert!(
19512                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
19513                "a probe started {:?} after the previous one, before it could finish",
19514                pair[1] - pair[0]
19515            );
19516        }
19517    }
19518
19519    #[tokio::test(start_paused = true)]
19520    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
19521        let drain = Drain::start(Answer::Silent);
19522        let ceiling = Duration::from_secs(2);
19523        let (drained, elapsed) = drain.run(ceiling).await;
19524        assert!(!drained, "an unanswered probe counts as busy");
19525        assert!(
19526            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
19527            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
19528        );
19529        assert_eq!(
19530            drain.probes().len(),
19531            1,
19532            "the unanswered probe holds the only slot until the ceiling"
19533        );
19534    }
19535
19536    /// Every event logged while it is installed, as `message` plus fields.
19537    #[derive(Clone, Default)]
19538    struct LogCapture {
19539        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
19540    }
19541
19542    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
19543        fn on_event(
19544            &self,
19545            event: &tracing::Event<'_>,
19546            _context: tracing_subscriber::layer::Context<'_, S>,
19547        ) {
19548            struct Fields(BTreeMap<String, String>);
19549            impl tracing::field::Visit for Fields {
19550                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
19551                    self.0
19552                        .insert(field.name().to_string(), format!("{value:?}"));
19553                }
19554            }
19555            let mut fields = Fields(BTreeMap::new());
19556            event.record(&mut fields);
19557            self.events.lock().unwrap().push(fields.0);
19558        }
19559    }
19560
19561    impl LogCapture {
19562        fn install(&self) -> tracing::subscriber::DefaultGuard {
19563            use tracing_subscriber::layer::SubscriberExt;
19564            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
19565        }
19566
19567        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
19568            self.events
19569                .lock()
19570                .unwrap()
19571                .iter()
19572                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
19573                .cloned()
19574                .collect()
19575        }
19576    }
19577
19578    /// A module that declares a busy gauge but leaves it out of its health
19579    /// answers holds a drain to its ceiling. The timeout line must say so by
19580    /// name, and the drain must warn about the missing gauge once, not on
19581    /// every probe.
19582    #[tokio::test(start_paused = true)]
19583    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
19584        let capture = LogCapture::default();
19585        let _guard = capture.install();
19586        let drain = Drain::start(Answer::OmitsGauge);
19587        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
19588        assert!(!drained, "a gauge never reported cannot read quiet");
19589        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
19590
19591        let missing = capture.with_message("lacks a declared busy gauge");
19592        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
19593        assert_eq!(
19594            missing[0].get("missing_gauges").map(String::as_str),
19595            Some(GAUGE)
19596        );
19597
19598        warn_drain_timed_out(
19599            MODULE_ID,
19600            Duration::from_secs(2),
19601            RouteCloseReason::Restart,
19602            &Default::default(),
19603            &trace,
19604        );
19605        let timeouts = capture.with_message("route drain timed out");
19606        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
19607        let gauges = &timeouts[0]["busy_gauges"];
19608        assert!(
19609            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
19610            "the timeout line must name the gauge and why it read busy: {gauges}"
19611        );
19612        assert!(
19613            gauges.ends_with("; no value ever read"),
19614            "no value was ever read for the gauge: {gauges}"
19615        );
19616        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
19617    }
19618
19619    /// The trace names a busy value and when it was read. For a probe that
19620    /// never got an answer, it names the probe's error kind and says that no
19621    /// value was ever read.
19622    #[tokio::test(start_paused = true)]
19623    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
19624        let busy = Drain::start(Answer::BusyUntil {
19625            until: Instant::now() + Duration::from_secs(3600),
19626            latency: Duration::ZERO,
19627        });
19628        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
19629        let rendered = trace.to_string();
19630        assert!(
19631            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
19632                && rendered.contains("; last value non_zero(1) @"),
19633            "{rendered}"
19634        );
19635
19636        let silent = Drain::start(Answer::Silent);
19637        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
19638        // The probe is capped at the 1 s ceiling, and its reading is stamped
19639        // when it gave up.
19640        assert_eq!(
19641            trace.to_string(),
19642            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
19643        );
19644        assert_eq!(trace.probes, 1);
19645    }
19646
19647    /// A gauge value that is not a count reads busy and is shown as sent.
19648    #[test]
19649    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
19650        let gauges = [GAUGE.to_string()];
19651        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
19652        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
19653        assert_eq!(
19654            busy_gauge_observation(&verdicts),
19655            BusyGaugeObservation::Busy
19656        );
19657        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
19658        assert_eq!(
19659            busy_gauge_observation(&zero),
19660            BusyGaugeObservation::Quiescent
19661        );
19662        let none = gauge_verdicts(None, &gauges);
19663        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
19664    }
19665}
19666
19667#[cfg(test)]
19668mod health_evidence_tests {
19669    use super::{HealthProbeError, HealthProbeEvidence};
19670    use std::collections::HashSet;
19671
19672    /// The evidential asymmetry, asserted rather than described.
19673    ///
19674    /// Exactly ONE observation is proof a module cannot serve, and the one that
19675    /// fires under CPU starvation is not it. Before the split, all fifteen
19676    /// construction sites collapsed into a single String, so a timeout carried the
19677    /// same weight as a dead lane -- which is how a healthy module was restarted
19678    /// three times in one day.
19679    #[test]
19680    fn only_a_dead_lane_is_proof_of_death() {
19681        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
19682        // Three non-proof classes, each for a different reason: silence is
19683        // consistent with health, a bad answer proves the module ALIVE, and a
19684        // daemon-side fault never reached the module at all.
19685        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
19686        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
19687        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
19688    }
19689
19690    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
19691    ///
19692    /// A shared label renders two different observations identically in the line an
19693    /// operator reads after an unexplained restart -- the exact confusion this
19694    /// change removes.
19695    #[test]
19696    fn every_evidence_class_has_a_distinct_label() {
19697        let labels = [
19698            HealthProbeError::lane_dead("").label(),
19699            HealthProbeError::no_answer("").label(),
19700            HealthProbeError::bad_answer("").label(),
19701            HealthProbeError::misconfigured("").label(),
19702        ];
19703        let unique: HashSet<_> = labels.iter().collect();
19704        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
19705    }
19706
19707    /// The class is additional information, not a replacement.
19708    ///
19709    /// An operator needs both "this was silence" and the specific text saying how
19710    /// long we waited; a classification that swallowed the message would trade one
19711    /// missing distinction for another.
19712    #[test]
19713    fn classification_preserves_the_original_message() {
19714        let err = HealthProbeError::no_answer("module did not answer within 5s");
19715        assert_eq!(err.to_string(), "module did not answer within 5s");
19716        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19717    }
19718}
19719
19720#[cfg(test)]
19721mod health_tombstone_tests {
19722    use std::{path::PathBuf, sync::Arc, time::Duration};
19723
19724    use subc_protocol::{
19725        manifest::Concurrency,
19726        session::{HealthStatus, ModuleControlResponse},
19727    };
19728    use tokio::sync::mpsc;
19729
19730    use super::{
19731        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
19732        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
19733    };
19734    use crate::{
19735        control::ControlHandler,
19736        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
19737        registry::{ConnectionId, Registry},
19738        router::FrameSink,
19739    };
19740
19741    struct ProbeHarness {
19742        spec: ModuleSpec,
19743        runtime: SupervisorRuntimeConfig,
19744        forwarding: Arc<ForwardingTable>,
19745        module_connection: ConnectionId,
19746        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19747        handler: ControlHandler,
19748        module: super::SupervisedModule,
19749    }
19750
19751    fn probe_harness() -> ProbeHarness {
19752        let registry = Arc::new(Registry::default());
19753        let forwarding = Arc::new(ForwardingTable::default());
19754        let supervisor_handle = super::SupervisorHandle::new();
19755        let health = HealthConfig {
19756            http: None,
19757            cadence: Duration::from_secs(30),
19758            deadline: Duration::from_secs(5),
19759            failure_threshold: 3,
19760            on_degraded: HealthAction::Report,
19761            on_failing: HealthAction::Report,
19762            critical: false,
19763        };
19764        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
19765            .with_forwarding(Arc::clone(&forwarding))
19766            .with_handle(supervisor_handle.clone())
19767            .with_health_config(health);
19768        let spec = ModuleSpec {
19769            script: None,
19770            module_id: "late-health-module".to_string(),
19771            program: PathBuf::from("disabled-module"),
19772            args: Vec::new(),
19773            env: Vec::new(),
19774            reserved: false,
19775            reserved_prefixes: Vec::new(),
19776            protocol: ModuleProtocol::Subc,
19777            overlap: Default::default(),
19778            carry_containment: false,
19779        };
19780        let module = supervisor
19781            .supervise_configured(spec.clone(), false)
19782            .unwrap();
19783        let runtime = supervisor.runtime_config();
19784        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
19785            .with_supervisor(supervisor_handle);
19786        let module_connection = ConnectionId::new(700);
19787        let (module_tx, module_rx) = mpsc::channel(8);
19788        forwarding
19789            .register_module_connection(
19790                module_connection,
19791                spec.module_id.clone(),
19792                subc_protocol::PROTOCOL_VERSION,
19793                Concurrency::ModuleManaged,
19794                FrameSink::new(module_tx),
19795            )
19796            .unwrap();
19797
19798        ProbeHarness {
19799            spec,
19800            runtime,
19801            forwarding,
19802            module_connection,
19803            module_rx,
19804            handler,
19805            module,
19806        }
19807    }
19808
19809    async fn finish_after(
19810        harness: &mut ProbeHarness,
19811        stall: Duration,
19812    ) -> ModuleControlRpcCompletion {
19813        assert!(stall > harness.runtime.health.deadline);
19814        let deadline = harness.runtime.health.deadline;
19815        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19816        let answer = async {
19817            let frame = harness.module_rx.recv().await.expect("health.check frame");
19818            tokio::time::advance(deadline).await;
19819            tokio::task::yield_now().await;
19820            tokio::time::advance(stall - deadline).await;
19821            harness
19822                .forwarding
19823                .complete_module_control_rpc(
19824                    harness.module_connection,
19825                    frame.header.corr,
19826                    Some("health.check"),
19827                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19828                        status: HealthStatus::Ok,
19829                        detail: None,
19830                        metrics: None,
19831                    }),
19832                )
19833                .unwrap()
19834        };
19835        let (probe_result, completion) = tokio::join!(probe, answer);
19836        let err = probe_result.expect_err("probe must miss its deadline");
19837        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19838        completion
19839    }
19840
19841    async fn time_out_without_answer(harness: &mut ProbeHarness) {
19842        let deadline = harness.runtime.health.deadline;
19843        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19844        let exhaust_deadline = async {
19845            let _frame = harness.module_rx.recv().await.expect("health.check frame");
19846            tokio::time::advance(deadline).await;
19847            tokio::task::yield_now().await;
19848        };
19849        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
19850        let err = probe_result.expect_err("probe must miss its deadline");
19851        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19852    }
19853
19854    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
19855        let registry = Arc::clone(&harness.module.inner.registry);
19856        let snapshot = Arc::clone(&harness.module.inner.snapshot);
19857        let process_liveness = super::SupervisorProcessLiveness::default();
19858        let mut child = None;
19859        let cycle = super::run_health_probe_cycle(
19860            &harness.spec,
19861            &harness.runtime,
19862            &registry,
19863            &process_liveness,
19864            &snapshot,
19865            &mut child,
19866        );
19867        let peer = async {
19868            let frame = harness.module_rx.recv().await.expect("health.check frame");
19869            if answer {
19870                harness
19871                    .forwarding
19872                    .complete_module_control_rpc(
19873                        harness.module_connection,
19874                        frame.header.corr,
19875                        Some("health.check"),
19876                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19877                            status: HealthStatus::Ok,
19878                            detail: None,
19879                            metrics: Some(serde_json::json!({"ready": true})),
19880                        }),
19881                    )
19882                    .unwrap();
19883            } else {
19884                tokio::time::advance(harness.runtime.health.deadline).await;
19885                tokio::task::yield_now().await;
19886            }
19887        };
19888        tokio::join!(cycle, peer);
19889    }
19890
19891    #[tokio::test(start_paused = true)]
19892    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
19893        let mut harness = probe_harness();
19894        // Drive the probe cycle directly with an in-memory wire peer. Stop the
19895        // disabled module's monitor so only this test owns lifecycle transitions;
19896        // no OS process is launched, and a restart is observed at scheduling.
19897        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
19898        monitor.abort();
19899        let _ = monitor.await;
19900        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
19901            state.enabled = true;
19902            state.state = super::ModuleState::Running;
19903            state.process_alive = true;
19904        })
19905        .unwrap();
19906
19907        run_probe_cycle(&mut harness, true).await;
19908        assert_eq!(
19909            harness.module.status().unwrap().health.status,
19910            super::SupervisorHealthStatus::Ok
19911        );
19912
19913        for failures in 1..harness.runtime.health.failure_threshold {
19914            run_probe_cycle(&mut harness, false).await;
19915            let status = harness.module.status().unwrap();
19916            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19917            assert_eq!(status.health.consecutive_failures, failures);
19918            assert!(status.health.last_probe_ms.is_some());
19919            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
19920            assert!(status.health.metrics.is_none());
19921            assert_eq!(status.state, super::ModuleState::Running);
19922            assert!(status.process_alive);
19923            assert_eq!(status.restart_count, 0);
19924            assert_eq!(status.lifetime_restarts, 0);
19925            assert!(status.health.last_action.is_none());
19926            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19927        }
19928
19929        run_probe_cycle(&mut harness, true).await;
19930        let recovered = harness.module.status().unwrap();
19931        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
19932        assert_eq!(recovered.health.consecutive_failures, 0);
19933        assert!(recovered.health.detail.is_none());
19934        assert_eq!(
19935            recovered.health.metrics,
19936            Some(serde_json::json!({"ready": true}))
19937        );
19938        assert_eq!(recovered.lifetime_restarts, 0);
19939
19940        for failures in 1..=harness.runtime.health.failure_threshold {
19941            run_probe_cycle(&mut harness, false).await;
19942            let status = harness.module.status().unwrap();
19943            assert_eq!(status.health.consecutive_failures, failures);
19944            if failures < harness.runtime.health.failure_threshold {
19945                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19946                assert_eq!(status.state, super::ModuleState::Running);
19947                assert_eq!(status.lifetime_restarts, 0);
19948                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19949            } else {
19950                assert_eq!(
19951                    status.health.status,
19952                    super::SupervisorHealthStatus::Unresponsive
19953                );
19954                assert_eq!(status.state, super::ModuleState::Restarting);
19955                assert_eq!(status.restart_count, 1);
19956                assert_eq!(status.lifetime_restarts, 1);
19957                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
19958                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
19959            }
19960        }
19961    }
19962
19963    #[tokio::test(start_paused = true)]
19964    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
19965        let mut harness = probe_harness();
19966
19967        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
19968        let first_latency = match &first {
19969            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19970            other => panic!("late answer was not retained: {other:?}"),
19971        };
19972        assert!(harness.handler.observe_module_control_completion(first));
19973
19974        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
19975        let second_latency = match &second {
19976            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19977            other => panic!("late answer was not retained: {other:?}"),
19978        };
19979        assert!(harness.handler.observe_module_control_completion(second));
19980
19981        assert_eq!(first_latency, Duration::from_secs(8));
19982        assert_eq!(
19983            second_latency - first_latency,
19984            Duration::from_secs(3),
19985            "latency must grow linearly with the additional stall"
19986        );
19987        let health = harness.module.status().unwrap().health;
19988        assert_eq!(health.late_answer_count, 2);
19989        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
19990    }
19991
19992    /// A module that answers every probe late must never march to the kill
19993    /// threshold: the late answer proves it is alive, so it must clear the miss
19994    /// streak the timeout recorded. Without the reset, a CPU-starved module
19995    /// that serves every probe seconds past the deadline accumulates
19996    /// `consecutive_failures` to the threshold and is killed — the exact
19997    /// sequence from the 2026-08-14 aft disable, where the daemon logged
19998    /// "proves the module is alive" five times while counting five misses.
19999    #[tokio::test(start_paused = true)]
20000    async fn late_answer_clears_the_consecutive_failure_streak() {
20001        let mut harness = probe_harness();
20002
20003        // Timeout recorded first: the probe path saw no answer in time.
20004        time_out_without_answer(&mut harness).await;
20005        harness
20006            .module
20007            .record_health_probe_failure_for_test("[no-answer] test miss")
20008            .unwrap();
20009        assert_eq!(
20010            harness.module.status().unwrap().health.consecutive_failures,
20011            1,
20012            "precondition: the miss must be on the streak before the late answer"
20013        );
20014
20015        // The stalled reply then lands: proof of life.
20016        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
20017        assert!(matches!(
20018            late,
20019            ModuleControlRpcCompletion::LateHealthAnswer { .. }
20020        ));
20021        assert!(harness.handler.observe_module_control_completion(late));
20022
20023        let health = harness.module.status().unwrap().health;
20024        assert_eq!(
20025            health.consecutive_failures, 0,
20026            "a late answer is an answer: the streak must reset"
20027        );
20028        assert_eq!(health.late_answer_count, 1);
20029    }
20030
20031    #[tokio::test(start_paused = true)]
20032    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
20033        let mut harness = probe_harness();
20034
20035        for _ in 0..20 {
20036            time_out_without_answer(&mut harness).await;
20037            assert_eq!(
20038                harness.forwarding.health_probe_tombstone_count().unwrap(),
20039                1
20040            );
20041        }
20042    }
20043}
20044
20045#[cfg(test)]
20046mod child_env_tests {
20047    #[cfg(windows)]
20048    use super::{
20049        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
20050    };
20051    #[cfg(windows)]
20052    use std::sync::{Arc, OnceLock};
20053    #[cfg(windows)]
20054    #[test]
20055    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
20056        let spec = spec(Vec::new());
20057        let mut command = Command::new("unused");
20058        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
20059            .unwrap()
20060            .unwrap();
20061        let name = command
20062            .as_std()
20063            .get_envs()
20064            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
20065            .and_then(|(_, value)| value)
20066            .unwrap();
20067        assert_eq!(name, OsStr::new(handoff.name()));
20068        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
20069        assert!(command.as_std().get_envs().any(|(key, value)| key
20070            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
20071            && value == Some(OsStr::new("env"))));
20072        assert!(command
20073            .as_std()
20074            .get_envs()
20075            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
20076    }
20077
20078    #[cfg(windows)]
20079    #[test]
20080    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
20081        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
20082        snapshot.process_alive = true;
20083        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
20084        assert_eq!(
20085            observed_launch_nonce_source(&snapshot).as_deref(),
20086            Some("env")
20087        );
20088        let consumed = Arc::new(OnceLock::new());
20089        snapshot.pipe_consumed = Some(consumed.clone());
20090        assert_eq!(
20091            observed_launch_nonce_source(&snapshot).as_deref(),
20092            Some("env")
20093        );
20094        consumed.set(()).unwrap();
20095        assert_eq!(
20096            observed_launch_nonce_source(&snapshot).as_deref(),
20097            Some("pipe")
20098        );
20099        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
20100        assert_eq!(
20101            observed_launch_nonce_source(&snapshot).as_deref(),
20102            Some("env")
20103        );
20104        clear_current_process_facts(&mut snapshot);
20105        assert_eq!(observed_launch_nonce_source(&snapshot), None);
20106    }
20107    use super::{
20108        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
20109        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
20110        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
20111    };
20112    use std::{ffi::OsStr, path::PathBuf};
20113    use tokio::process::Command;
20114
20115    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
20116        ModuleSpec {
20117            script: None,
20118            module_id: "env-plan".to_string(),
20119            program: PathBuf::from("/nonexistent"),
20120            args: Vec::new(),
20121            env,
20122            reserved: false,
20123            reserved_prefixes: Vec::new(),
20124            protocol: ModuleProtocol::Subc,
20125            overlap: Default::default(),
20126            carry_containment: false,
20127        }
20128    }
20129
20130    #[test]
20131    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
20132        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
20133        let digest = LaunchSpecDigest::from_spec(&base);
20134        let mut reordered = base.clone();
20135        reordered.env.reverse();
20136        reordered.module_id = "different-id".into();
20137        reordered.reserved = true;
20138        reordered.overlap = Default::default();
20139        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
20140        let mut delimiter_collision = base.clone();
20141        delimiter_collision.env[0] = ("A=B".into(), "C".into());
20142        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
20143        let mut changed = base.clone();
20144        changed.protocol = ModuleProtocol::None;
20145        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20146        changed = base.clone();
20147        changed.args.push("--flag".into());
20148        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20149        changed = base.clone();
20150        changed.program = PathBuf::from("/other");
20151        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20152        changed = base.clone();
20153        changed.env[1].1 = "other".into();
20154        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20155        assert!(!format!("{digest:?}").contains("B=C"));
20156        assert!(!format!("{digest:?}").contains("[u8"));
20157    }
20158
20159    #[cfg(unix)]
20160    #[test]
20161    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
20162        use std::os::unix::ffi::OsStringExt;
20163        let mut left = spec(Vec::new());
20164        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
20165        let mut right = left.clone();
20166        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
20167        assert_eq!(
20168            left.program.to_string_lossy(),
20169            right.program.to_string_lossy()
20170        );
20171        assert_ne!(
20172            LaunchSpecDigest::from_spec(&left),
20173            LaunchSpecDigest::from_spec(&right)
20174        );
20175    }
20176
20177    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
20178    /// one still gets its own.
20179    ///
20180    /// This is the narrow goal `env_clear()` was reached for, and the reason the
20181    /// fix is `env_remove` rather than deleting the line: an operator's ambient
20182    /// filter silently becoming an unconfigured module's log level is a real
20183    /// defect, just a much smaller one than clearing the environment.
20184    ///
20185    /// Asserted on the command plan rather than a spawned child because proving
20186    /// the ABSENCE of an inherited variable needs the parent's environment
20187    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
20188    /// removal as `(key, None)`, which is exactly the distinction wanted: not
20189    /// "absent because nobody set it" but "explicitly unset for the child".
20190    #[test]
20191    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
20192        let mut command = Command::new("/nonexistent");
20193        apply_child_env(&mut command, &spec(Vec::new()));
20194        let removed = command
20195            .as_std()
20196            .get_envs()
20197            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
20198        assert!(
20199            removed,
20200            "ambient CK_LOG must be explicitly removed for an unconfigured module"
20201        );
20202
20203        let mut configured = Command::new("/nonexistent");
20204        apply_child_env(
20205            &mut configured,
20206            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
20207        );
20208        let effective = configured
20209            .as_std()
20210            .get_envs()
20211            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
20212            .last()
20213            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
20214        assert_eq!(
20215            effective,
20216            Some(Some("debug".to_string())),
20217            "a module's configured CK_LOG must survive the ambient removal"
20218        );
20219    }
20220
20221    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
20222    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
20223    /// the same reason as the CK_LOG test above.
20224    ///
20225    /// The argument is the load-bearing half: a stock binary exits on an
20226    /// unknown flag before it listens, so with `--subc` appended the mode
20227    /// could not supervise the one process it exists for. Found by the first
20228    /// conformance run (nats-server: `flag provided but not defined: -subc`).
20229    #[test]
20230    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
20231        let connection_file = std::path::Path::new("/run/subc-connection.json");
20232        let handle = SupervisorHandle::new();
20233
20234        let mut none_spec = spec(Vec::new());
20235        none_spec.protocol = ModuleProtocol::None;
20236        let mut none = Command::new("/nonexistent");
20237        let none_handoff =
20238            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
20239                .expect("protocol-none spawn args apply");
20240        assert!(
20241            none_handoff.is_none(),
20242            "protocol:none spawn must not receive a nonce descriptor"
20243        );
20244        #[cfg(windows)]
20245        assert!(!none.as_std().get_envs().any(|(key, value)| key
20246            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
20247            && value.is_some()));
20248        assert!(
20249            !none.as_std().get_envs().any(|(key, value)| key
20250                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
20251                && value.is_some()),
20252            "protocol:none spawn must not name a nonce descriptor"
20253        );
20254        let none_args: Vec<String> = none
20255            .as_std()
20256            .get_args()
20257            .map(|a| a.to_string_lossy().into_owned())
20258            .collect();
20259        assert!(
20260            !none_args.iter().any(|a| a == SUBC_ARG),
20261            "protocol:none argv must not carry --subc; got {none_args:?}"
20262        );
20263        let none_has_nonce = none
20264            .as_std()
20265            .get_envs()
20266            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
20267        assert!(
20268            !none_has_nonce,
20269            "protocol:none spawn must not receive a launch nonce"
20270        );
20271        let none_has_module_id = none
20272            .as_std()
20273            .get_envs()
20274            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
20275        assert!(
20276            none_has_module_id,
20277            "SUBC_MODULE_ID is inert and stays on every path"
20278        );
20279        assert!(
20280            handle.spawn_nonce(&none_spec.module_id).is_none(),
20281            "no nonce record for a process that will never present one"
20282        );
20283
20284        // Control: the subc-wire path is unchanged by the branch above.
20285        let wire_spec = spec(Vec::new());
20286        let mut wire = Command::new("/nonexistent");
20287        let wire_handoff =
20288            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
20289                .expect("subc-wire spawn args apply");
20290        let wire_fd_env = wire
20291            .as_std()
20292            .get_envs()
20293            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
20294            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
20295        #[cfg(unix)]
20296        assert_eq!(
20297            wire_fd_env,
20298            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
20299            "a subc-wire spawn names the pipe it will receive at descriptor 3"
20300        );
20301        #[cfg(windows)]
20302        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
20303        #[cfg(not(any(unix, windows)))]
20304        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
20305        let wire_args: Vec<String> = wire
20306            .as_std()
20307            .get_args()
20308            .map(|a| a.to_string_lossy().into_owned())
20309            .collect();
20310        assert_eq!(
20311            wire_args,
20312            vec![
20313                SUBC_ARG.to_string(),
20314                connection_file.to_string_lossy().into_owned()
20315            ],
20316            "a subc-wire spawn still carries --subc <path>"
20317        );
20318        assert_eq!(
20319            wire.as_std()
20320                .get_envs()
20321                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
20322            !cfg!(unix),
20323            "only Windows supplies the environment nonce"
20324        );
20325        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
20326    }
20327
20328    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
20329    /// spec tries to set it; only a swap candidate carries it.
20330    ///
20331    /// "Set it only on candidates" is not enough, because spawn applies the
20332    /// spec's env verbatim and the daemon's own environment is inherited: either
20333    /// could hand a plain restart the swap role, and a module reading it would
20334    /// warm on its long swap budget while callers wait. Asserted as an explicit
20335    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
20336    /// test above gives.
20337    #[test]
20338    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
20339        let role = |command: &Command| {
20340            command
20341                .as_std()
20342                .get_envs()
20343                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
20344                .last()
20345                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
20346        };
20347        let forged = spec(vec![(
20348            SUBC_SPAWN_ROLE_ENV.to_string(),
20349            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
20350        )]);
20351
20352        let mut plain = Command::new("/nonexistent");
20353        apply_child_env(&mut plain, &forged);
20354        apply_spawn_role(&mut plain, SpawnRole::Plain);
20355        assert_eq!(
20356            role(&plain),
20357            Some(None),
20358            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
20359        );
20360
20361        let mut candidate = Command::new("/nonexistent");
20362        apply_child_env(&mut candidate, &spec(Vec::new()));
20363        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
20364        assert_eq!(
20365            role(&candidate),
20366            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
20367        );
20368    }
20369
20370    /// Daemon-private capture retention keys never reach the child.
20371    ///
20372    /// cortexkit-log exposes retention as a Rust struct with no environment
20373    /// names, so these entries are supervisor metadata. Passing them through
20374    /// would invent a public child-process contract by accident.
20375    #[test]
20376    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
20377        let mut command = Command::new("/nonexistent");
20378        apply_child_env(
20379            &mut command,
20380            &spec(vec![
20381                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
20382                ("KEPT".to_string(), "yes".to_string()),
20383            ]),
20384        );
20385        let keys: Vec<String> = command
20386            .as_std()
20387            .get_envs()
20388            .filter(|(_, value)| value.is_some())
20389            .map(|(key, _)| key.to_string_lossy().into_owned())
20390            .collect();
20391        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
20392        assert!(
20393            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
20394            "daemon-private capture key leaked to the child: {keys:?}"
20395        );
20396    }
20397}
20398
20399#[cfg(test)]
20400mod jitter_tests {
20401    use super::jittered_health_delay;
20402    use std::{collections::HashSet, time::Duration};
20403
20404    /// Module ids drawn from a real fleet, so the dispersal claim is about names
20405    /// that actually occur rather than invented ones.
20406    ///
20407    /// This is a SAMPLE, not a registry: the property under test is that distinct
20408    /// ids disperse, which holds for any set of distinct strings. Several entries
20409    /// are already historical (modules get renamed), and that costs nothing here --
20410    /// but it means a reader must not mistake this for the live module set, and a
20411    /// rename sweep will match it without there being anything to change.
20412    const FLEET: [&str; 14] = [
20413        "aft",
20414        "alfonso-core",
20415        "magic-context",
20416        "broca",
20417        "thalamus",
20418        "quota",
20419        "engram",
20420        "plexus",
20421        "cerebellum",
20422        "astrocyte",
20423        "synapse",
20424        "subc-mcp",
20425        "cortexkit-credentials",
20426        "subc-federation",
20427    ];
20428
20429    /// Probes must not converge after a fleet-wide restart.
20430    ///
20431    /// This is the property the jitter exists for: every module reconnects at
20432    /// once, and without dispersal all fourteen would then probe on the same
20433    /// tick forever. Nothing failed visibly when this went untested -- a
20434    /// convergent fleet still probes correctly, just in a burst, so the symptom
20435    /// is a periodic load spike that looks like whatever else is running.
20436    #[test]
20437    fn probe_delays_disperse_across_the_fleet() {
20438        let cadence = Duration::from_secs(30);
20439        let delays: HashSet<Duration> = FLEET
20440            .iter()
20441            .map(|id| jittered_health_delay(id, 0, cadence))
20442            .collect();
20443        assert_eq!(
20444            delays.len(),
20445            FLEET.len(),
20446            "every supervised module must land on its own probe offset"
20447        );
20448    }
20449
20450    /// The offset may only ever DELAY a probe, never bring it forward.
20451    ///
20452    /// A delay below the cadence would probe a module more often than
20453    /// configured, which is the opposite of what an operator asked for and
20454    /// would tighten the failure budget without anyone changing it.
20455    #[test]
20456    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
20457        let cadence = Duration::from_secs(30);
20458        let span = cadence / 10;
20459        for id in FLEET {
20460            for probe_index in 0..8 {
20461                let delay = jittered_health_delay(id, probe_index, cadence);
20462                assert!(
20463                    delay >= cadence,
20464                    "{id}#{probe_index}: jitter must not shorten the cadence"
20465                );
20466                assert!(
20467                    delay < cadence + span,
20468                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
20469                );
20470            }
20471        }
20472    }
20473
20474    /// A module keeps its offset across daemon restarts.
20475    ///
20476    /// The delay is derived rather than randomised precisely so a restart does
20477    /// not re-roll every module into a fresh chance of collision. A random
20478    /// source would satisfy the dispersal test above and quietly lose this.
20479    #[test]
20480    fn a_module_offset_is_stable_across_restarts() {
20481        let cadence = Duration::from_secs(30);
20482        for id in FLEET {
20483            assert_eq!(
20484                jittered_health_delay(id, 0, cadence),
20485                jittered_health_delay(id, 0, cadence),
20486                "{id}: the same module and probe index must produce the same offset"
20487            );
20488        }
20489    }
20490
20491    /// A zero cadence disables probing rather than producing a busy loop.
20492    #[test]
20493    fn zero_cadence_yields_zero_delay() {
20494        assert_eq!(
20495            jittered_health_delay("aft", 0, Duration::ZERO),
20496            Duration::ZERO
20497        );
20498    }
20499}
20500
20501#[cfg(all(test, target_os = "linux"))]
20502mod cgroup_placement_tests {
20503    use super::{
20504        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
20505        SuperviseError, SupervisedChild,
20506    };
20507    use crate::stderr_tail::{StderrRing, StderrTailConfig};
20508    use cortexkit_test_support::ScratchDir;
20509    use std::{
20510        fs, io,
20511        path::{Path, PathBuf},
20512        sync::{Arc, Mutex},
20513    };
20514    use tokio::process::Command;
20515
20516    #[tokio::test]
20517    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
20518        use super::*;
20519        let dir = ScratchDir::new("unique-spawn-cgroups");
20520        let root = PathBuf::from(format!(
20521            "/sys/fs/cgroup/subc-unique-test-{}-{}",
20522            std::process::id(),
20523            unix_ms_now()
20524        ));
20525        if let Err(error) = fs::create_dir(&root) {
20526            assert!(
20527                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
20528                "required cgroup test cannot execute: {error}"
20529            );
20530            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
20531            return;
20532        }
20533        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
20534        let group_count = || {
20535            fs::read_dir(root.join("subc-modules"))
20536                .unwrap()
20537                .map(|entry| entry.unwrap().file_type().unwrap())
20538                .filter(|kind| kind.is_dir())
20539                .count()
20540        };
20541        let supervisor =
20542            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
20543                .with_cgroup_placement(Some(placement.clone()));
20544        let runtime = supervisor.runtime_config();
20545        let mut spec = ModuleSpec {
20546            script: None,
20547            module_id: "unique-spawn".into(),
20548            program: PathBuf::from("/bin/sleep"),
20549            args: vec!["60".into()],
20550            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
20551                .into_iter()
20552                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
20553                .collect(),
20554            reserved: false,
20555            reserved_prefixes: vec![],
20556            protocol: ModuleProtocol::None,
20557            overlap: Default::default(),
20558            carry_containment: false,
20559        };
20560        let spawn = |spec: &ModuleSpec| {
20561            spawn_child(
20562                spec,
20563                None,
20564                None,
20565                &runtime.stderr_ring,
20566                None,
20567                &runtime.child_roster,
20568                runtime.spawn_delivery(),
20569                Some(&placement),
20570                runtime.register_timeout,
20571            )
20572            .unwrap()
20573        };
20574        let mut live = spawn(&spec);
20575        for _ in 0..3 {
20576            // A new process can enter the old slot while retirement is pending.
20577            let next = spawn(&spec);
20578            assert_ne!(live.module_id, next.module_id);
20579            live.start_kill().unwrap();
20580            live.wait().await.unwrap();
20581            live = next;
20582            assert!(
20583                live.child.try_wait().unwrap().is_none(),
20584                "retiring the old slot must not kill the replacement"
20585            );
20586            assert_eq!(
20587                group_count(),
20588                1,
20589                "only the live spawn's cgroup should remain"
20590            );
20591        }
20592        supervisor.begin_daemon_shutdown();
20593        let reap = tokio::spawn(async move {
20594            live.wait().await.unwrap();
20595        });
20596        supervisor
20597            .end_children_for_daemon_shutdown(false, std::future::pending())
20598            .await;
20599        reap.await.unwrap();
20600        assert_eq!(group_count(), 0);
20601        // A normal exit uses the same tree-cleanup path as a killed spawn.
20602        spec.program = PathBuf::from("/bin/true");
20603        spec.args.clear();
20604        let fresh_roster = ChildRoster::default();
20605        let mut short = spawn_child(
20606            &spec,
20607            None,
20608            None,
20609            &runtime.stderr_ring,
20610            None,
20611            &fresh_roster,
20612            None,
20613            Some(&placement),
20614            runtime.register_timeout,
20615        )
20616        .unwrap();
20617        short.wait().await.unwrap();
20618        assert_eq!(group_count(), 0);
20619        spec.module_id = "_".repeat(255);
20620        let mut long_id = spawn_child(
20621            &spec,
20622            None,
20623            None,
20624            &runtime.stderr_ring,
20625            None,
20626            &fresh_roster,
20627            None,
20628            Some(&placement),
20629            runtime.register_timeout,
20630        )
20631        .unwrap();
20632        long_id.wait().await.unwrap();
20633        assert_eq!(
20634            group_count(),
20635            0,
20636            "valid long module IDs must not exceed cgroup NAME_MAX"
20637        );
20638        fs::remove_dir(root.join("subc-modules")).unwrap();
20639        fs::remove_dir(root).unwrap();
20640        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
20641    }
20642
20643    #[test]
20644    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
20645        let path = Path::new("/definitely-missing-subc-cgroup");
20646        let mut command = Command::new("true");
20647        let error = apply_cgroup_placement(
20648            &mut command,
20649            &ModuleSpec {
20650                script: None,
20651                module_id: "broken-cgroup".to_string(),
20652                program: PathBuf::from("true"),
20653                args: Vec::new(),
20654                env: Vec::new(),
20655                reserved: false,
20656                reserved_prefixes: Vec::new(),
20657                protocol: ModuleProtocol::Subc,
20658                overlap: Default::default(),
20659                carry_containment: false,
20660            },
20661            path,
20662        )
20663        .expect_err("a parent cgroup open failure must reject the supervised spawn");
20664        let reason = error.to_string();
20665
20666        assert!(
20667            matches!(error, SuperviseError::Cgroup { .. }),
20668            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
20669        );
20670        assert!(
20671            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
20672            "parent cgroup open failure must name cgroup.procs: {reason}"
20673        );
20674    }
20675
20676    #[tokio::test]
20677    async fn reaping_a_child_removes_its_empty_module_cgroup() {
20678        let root = ScratchDir::new("supervisor-reap-cgroup");
20679        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20680        let placement = subc_cgroup::prepare_at(&root)
20681            .expect("prepare scratch cgroup root")
20682            .expect("scratch root has a cgroup.procs marker");
20683        let module_id = "reaped-module";
20684        let module = placement
20685            .module_path(module_id)
20686            .expect("create scratch module cgroup");
20687        let child = Command::new("true")
20688            .env("XDG_DATA_HOME", root.path())
20689            .env("XDG_RUNTIME_DIR", root.path())
20690            .env("XDG_CONFIG_HOME", root.path())
20691            .spawn()
20692            .expect("spawn short-lived child");
20693        let pid = child.id().expect("spawned child has pid");
20694        let mut child = SupervisedChild {
20695            registration_channel: None,
20696            registration_spawn: None,
20697            live_program_binding: None,
20698            registered_config_connection: None,
20699            admission_base: None,
20700            registration_declaration: None,
20701            _program_binding: None,
20702            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
20703            child: crate::supervise::containment::ProcessChild::plain(child, pid),
20704            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
20705            protocol: ModuleProtocol::Subc,
20706            module_id: module_id.to_string(),
20707            cgroup_placement: Some(placement),
20708            stdout_pump: None,
20709            stderr_pump: None,
20710            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
20711            spawned_at: tokio::time::Instant::now(),
20712            config_delivery: None,
20713            registration_deadline: None,
20714            registration_watch: None,
20715            spawned_at_ms: 0,
20716            spawned_from: PathBuf::from("true"),
20717            launch_spec_digest: LaunchSpecDigest([0; 32]),
20718            spawned_file_identity: None,
20719            process_start_time: None,
20720            process_identity: None,
20721            pid,
20722            roster_guard: None,
20723            #[cfg(target_os = "macos")]
20724            privacy_exec: None,
20725            spawn_failure: None,
20726            admission_kill_sent: false,
20727        };
20728
20729        child.wait().await.expect("reap short-lived child");
20730
20731        assert!(
20732            !module.exists(),
20733            "waiting for a non-carried child must remove its empty cgroup before returning"
20734        );
20735        child.finish_exit(false).await.unwrap();
20736    }
20737
20738    #[test]
20739    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
20740        let root = ScratchDir::new("supervisor-non-empty-cgroup");
20741        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20742        let placement = subc_cgroup::prepare_at(&root)
20743            .expect("prepare scratch cgroup root")
20744            .expect("scratch root has a cgroup.procs marker");
20745        let module = placement
20746            .module_path("surviving-module")
20747            .expect("create scratch module cgroup");
20748        fs::write(module.join("surviving-process"), b"still present")
20749            .expect("make scratch cgroup non-empty");
20750        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
20751
20752        remove_module_cgroup(&placement, "surviving-module");
20753
20754        let logs = crate::router::test_log::captured_logs(&logs);
20755        assert!(
20756            module.exists(),
20757            "failed removal must leave the cgroup intact"
20758        );
20759        assert!(
20760            logs.contains("could not remove module cgroup after process exit; continuing teardown")
20761                && logs.contains("surviving-module"),
20762            "best-effort removal must report the failure without returning it: {logs}"
20763        );
20764    }
20765
20766    #[test]
20767    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
20768        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
20769        let reason = SuperviseError::Spawn {
20770            program: PathBuf::from("/bin/true"),
20771            source: io::Error::from_raw_os_error(13),
20772            cgroup_path: Some(cgroup_path.clone()),
20773        }
20774        .to_string();
20775
20776        assert!(
20777            reason.contains(&cgroup_path.display().to_string()),
20778            "a pre_exec spawn failure must name the cgroup path: {reason}"
20779        );
20780    }
20781}
20782
20783#[cfg(test)]
20784mod spawn_subscriber_lag_tests {
20785    use super::*;
20786
20787    /// A subscriber whose connection stops draining is dropped once its frame
20788    /// channel fills. The client must learn that from a terminal Error frame
20789    /// after the frames already queued for it, not from a stream that simply
20790    /// goes quiet.
20791    #[tokio::test]
20792    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
20793        let feed = SpawnEventFeed::default();
20794        feed.configure_incarnation("lag-incarnation".to_string());
20795        // A one-slot connection queue that nobody reads until the emits are
20796        // done: the forwarder parks on it and the subscriber channel fills.
20797        let (tx, mut rx) = mpsc::channel(1);
20798        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
20799            .expect("subscribe");
20800        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
20801        for index in 0..emitted {
20802            feed.emit_spawned(&format!("lag-module-{index}"), None, 1000, 0);
20803            // Let the forwarder take what it can so the fill point is the
20804            // subscriber channel, not a scheduling accident.
20805            tokio::task::yield_now().await;
20806        }
20807        assert_eq!(
20808            feed.subscriber_count(),
20809            0,
20810            "the lagged subscriber must be removed"
20811        );
20812
20813        let mut data = Vec::new();
20814        let mut last = None;
20815        loop {
20816            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
20817                .await
20818                .expect("the forwarder must finish once the subscriber is dropped");
20819            let Some(outbound) = next else { break };
20820            let frame = outbound.frame;
20821            if frame.header.ty == FrameType::StreamData {
20822                assert!(last.is_none(), "no data may follow the terminal frame");
20823                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
20824                data.push(event.cursor.seq);
20825            } else {
20826                assert!(last.is_none(), "exactly one terminal frame");
20827                last = Some(frame);
20828            }
20829        }
20830        assert!(!data.is_empty(), "queued frames drain before the terminal");
20831        for pair in data.windows(2) {
20832            assert_eq!(
20833                pair[1],
20834                pair[0] + 1,
20835                "queued frames arrive dense and in order"
20836            );
20837        }
20838        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
20839        assert_eq!(terminal.header.ty, FrameType::Error);
20840        assert_eq!(terminal.header.corr, 7);
20841        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
20842        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
20843        let detail = body.detail.expect("lagged error carries detail");
20844        assert_eq!(
20845            detail["first_undelivered_cursor"]["seq"],
20846            data.last().unwrap() + 1,
20847            "the named cursor is the first event the subscriber did not receive"
20848        );
20849        assert_eq!(
20850            detail["first_undelivered_cursor"]["daemon_incarnation"],
20851            "lag-incarnation"
20852        );
20853    }
20854}
20855
20856#[cfg(test)]
20857mod terminal_history_read_concurrency_tests {
20858    use super::*;
20859    use crate::terminal_journal::read_pause;
20860    use cortexkit_test_support::ScratchDir;
20861    use std::sync::mpsc as std_mpsc;
20862
20863    fn journaled_ring(
20864        journal: &Arc<crate::terminal_journal::TerminalJournal>,
20865    ) -> Arc<Mutex<TerminalRing>> {
20866        Arc::new(Mutex::new(
20867            TerminalRing::new(TerminalRingConfig::default(), 1)
20868                .with_journal(Some(Arc::clone(journal))),
20869        ))
20870    }
20871
20872    fn crash(at_ms: u64) -> ExitReport {
20873        ExitReport {
20874            kind: ExitKind::Crash,
20875            code: Some(1),
20876            signal: None,
20877            at_ms,
20878        }
20879    }
20880
20881    /// Record an exit on another thread and report whether it finished within
20882    /// `bound`. The recorder thread is left running if it did not.
20883    fn record_within(
20884        module_id: &'static str,
20885        ring: &Arc<Mutex<TerminalRing>>,
20886        at_ms: u64,
20887        bound: Duration,
20888    ) -> bool {
20889        let ring = Arc::clone(ring);
20890        let (done, done_rx) = std_mpsc::channel();
20891        std::thread::spawn(move || {
20892            record_terminal(
20893                module_id,
20894                &ring,
20895                &SpawnEventFeed::default(),
20896                &crash(at_ms),
20897                TerminalDisposition::Restarting,
20898            );
20899            let _ = done.send(());
20900        });
20901        done_rx.recv_timeout(bound).is_ok()
20902    }
20903
20904    /// A history read in progress must not hold the journal writer (which every
20905    /// module's exit recording needs) or the module's own ring. Exits recorded
20906    /// while the read is paused complete promptly; the paused read answers as of
20907    /// the moment it started, and the next read has each exit exactly once.
20908    #[test]
20909    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
20910        let dir = ScratchDir::new("terminal-history-concurrent-read");
20911        let path = dir.join("terminals.jsonl");
20912        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
20913            path.clone(),
20914            "daemon".into(),
20915        ));
20916        let reader_ring = journaled_ring(&journal);
20917        let other_ring = journaled_ring(&journal);
20918        assert!(record_within(
20919            "reader-module",
20920            &reader_ring,
20921            10,
20922            Duration::from_secs(5)
20923        ));
20924
20925        let (started, release) = read_pause::install(&path);
20926        let reading = {
20927            let ring = Arc::clone(&reader_ring);
20928            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
20929        };
20930        started
20931            .recv_timeout(Duration::from_secs(5))
20932            .expect("the history read reached its pause");
20933
20934        let bound = Duration::from_secs(1);
20935        assert!(
20936            record_within("other-module", &other_ring, 20, bound),
20937            "another module's exit waited on a history read (journal writer held)"
20938        );
20939        assert!(
20940            record_within("reader-module", &reader_ring, 30, bound),
20941            "the read module's own exit waited on its history read (ring held)"
20942        );
20943
20944        drop(release);
20945        let paused = reading.join().unwrap();
20946        assert_eq!(
20947            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20948            vec![10],
20949            "an exit recorded after the read began lands in neither half of it"
20950        );
20951        assert_eq!(paused.journal_skipped_lines, 0);
20952        assert_eq!(paused.journal_read_errors, 0);
20953
20954        let after = durable_terminal_history_of(&reader_ring, "reader-module");
20955        assert_eq!(
20956            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20957            vec![10, 30],
20958            "the next read merges ring and journal with no duplicate"
20959        );
20960        assert_eq!(after.journal_skipped_lines, 0);
20961    }
20962}
20963
20964/// What a restart does with the exited process's output readers. These drive
20965/// the same settle functions the supervisor calls, with a reader the test
20966/// holds, so a reader that has not been scheduled by the bound is a controlled
20967/// input rather than something only a loaded machine produces.
20968#[cfg(test)]
20969mod stderr_settle_tests {
20970    use std::{
20971        future::Future,
20972        io,
20973        pin::Pin,
20974        sync::{Arc, Mutex},
20975        task::{Context, Poll},
20976        time::Duration,
20977    };
20978
20979    use tokio::{
20980        io::{AsyncRead, ReadBuf},
20981        sync::oneshot,
20982        time::Instant,
20983    };
20984
20985    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
20986    use crate::stderr_tail::{
20987        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
20988        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
20989    };
20990
20991    const BOUND: Duration = Duration::from_millis(250);
20992
20993    /// Yields `before`, then stays pending until the gate is released, then
20994    /// yields `after` and reaches EOF. The bytes after the gate were written
20995    /// by a process that has already exited; only the reader is behind.
20996    struct HeldReader {
20997        before: Option<Vec<u8>>,
20998        gate: Option<oneshot::Receiver<()>>,
20999        after: io::Cursor<Vec<u8>>,
21000    }
21001
21002    impl AsyncRead for HeldReader {
21003        fn poll_read(
21004            mut self: Pin<&mut Self>,
21005            cx: &mut Context<'_>,
21006            buf: &mut ReadBuf<'_>,
21007        ) -> Poll<io::Result<()>> {
21008            if let Some(bytes) = self.before.take() {
21009                buf.put_slice(&bytes);
21010                return Poll::Ready(Ok(()));
21011            }
21012            if let Some(gate) = self.gate.as_mut() {
21013                match Pin::new(gate).poll(cx) {
21014                    Poll::Pending => return Poll::Pending,
21015                    Poll::Ready(_) => self.gate = None,
21016                }
21017            }
21018            Pin::new(&mut self.after).poll_read(cx, buf)
21019        }
21020    }
21021
21022    struct DiscardSink;
21023
21024    impl OutputSink for DiscardSink {
21025        fn write_line(&mut self, _line: &[u8]) {}
21026    }
21027
21028    fn line(text: &str) -> TailEntry {
21029        TailEntry::Line {
21030            text: text.to_string(),
21031            truncated: false,
21032            at_ms: None,
21033        }
21034    }
21035
21036    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
21037        ring.lock().unwrap()
21038    }
21039
21040    /// Start a reader for a new process generation that delivers `before`
21041    /// immediately and `after` only once the returned sender fires (or is
21042    /// dropped).
21043    fn held_pump(
21044        ring: &Arc<Mutex<StderrRing>>,
21045        before: &str,
21046        after: &str,
21047    ) -> (StderrPump, oneshot::Sender<()>) {
21048        let generation = lock(ring).begin_process();
21049        let (release, gate) = oneshot::channel();
21050        let reader = HeldReader {
21051            before: Some(before.as_bytes().to_vec()),
21052            gate: Some(gate),
21053            after: io::Cursor::new(after.as_bytes().to_vec()),
21054        };
21055        let task = tokio::spawn(pump_stderr_to(
21056            reader,
21057            Arc::clone(ring),
21058            generation,
21059            DiscardSink,
21060        ));
21061        (StderrPump { task, generation }, release)
21062    }
21063
21064    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
21065        for _ in 0..1000 {
21066            if done(&lock(ring)) {
21067                return;
21068            }
21069            tokio::time::sleep(Duration::from_millis(1)).await;
21070        }
21071        panic!(
21072            "ring never reached the expected state: {:?}",
21073            lock(ring).snapshot(None, None)
21074        );
21075    }
21076
21077    #[tokio::test(start_paused = true)]
21078    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
21079        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21080        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
21081
21082        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
21083        let before_release = lock(&ring).snapshot(None, None);
21084        assert!(
21085            matches!(before_release.capture, CaptureState::Incomplete { .. }),
21086            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
21087        );
21088
21089        // The restart: the next process starts and writes before the old
21090        // reader catches up.
21091        let next = lock(&ring).begin_process();
21092        lock(&ring).push_line_from(next, "next process booting");
21093        release.send(()).unwrap();
21094        wait_until(&ring, |ring| {
21095            ring.snapshot(None, None).capture == CaptureState::Captured
21096        })
21097        .await;
21098
21099        assert_eq!(
21100            untimed(lock(&ring).snapshot(None, None).entries),
21101            vec![
21102                line("booting"),
21103                line("config error: missing storage"),
21104                TailEntry::ProcessStart,
21105                line("next process booting"),
21106            ],
21107            "the crash's last line must survive a slow reader and stay in the crashed process's section"
21108        );
21109    }
21110
21111    #[tokio::test(start_paused = true)]
21112    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
21113    ) {
21114        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21115        // `_held` is never fired: a descendant keeps the pipe open for the
21116        // whole test.
21117        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
21118
21119        let started = Instant::now();
21120        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
21121        assert_eq!(
21122            started.elapsed(),
21123            BOUND,
21124            "the restart must wait exactly the bound for a pipe that stays open, no longer"
21125        );
21126
21127        let next = lock(&ring).begin_process();
21128        lock(&ring).push_line_from(next, "next process booting");
21129        tokio::time::sleep(Duration::from_secs(60)).await;
21130
21131        let snapshot = lock(&ring).snapshot(None, None);
21132        match &snapshot.capture {
21133            CaptureState::Incomplete { reason } => assert!(
21134                reason.contains("had not reached EOF") && reason.contains("250ms"),
21135                "the reason must say what is missing and after how long: {reason}"
21136            ),
21137            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
21138        }
21139        assert_eq!(
21140            untimed(snapshot.entries),
21141            vec![
21142                line("parent exiting"),
21143                TailEntry::ProcessStart,
21144                line("next process booting"),
21145            ]
21146        );
21147    }
21148
21149    #[tokio::test(start_paused = true)]
21150    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
21151        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21152        let (pump, release) = held_pump(&ring, "one\n", "two\n");
21153        release.send(()).unwrap();
21154
21155        settle_stderr_pump("clean", &ring, pump, BOUND).await;
21156
21157        let snapshot = lock(&ring).snapshot(None, None);
21158        assert_eq!(snapshot.capture, CaptureState::Captured);
21159        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
21160    }
21161
21162    #[tokio::test(start_paused = true)]
21163    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
21164        const BURST: usize = 150;
21165        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
21166        let path = capture.join("two-pipe.stderr.log");
21167        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
21168        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21169        let generation = lock(&ring).begin_process();
21170        let stdout = (0..BURST)
21171            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
21172            .collect::<String>();
21173        let stderr = (0..BURST)
21174            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
21175            .collect::<String>();
21176        // Model a child that has written both bursts and exited. Hold the stdout
21177        // reader after one line and a partial prefix, as if pipe readiness or
21178        // capture-file writes had delayed it. The final line also needs EOF to
21179        // flush: neither complete nor partial buffered output may be discarded.
21180        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
21181        let (release, gate) = oneshot::channel();
21182        let reader = HeldReader {
21183            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
21184            gate: Some(gate),
21185            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
21186        };
21187        let (finished, completion) = oneshot::channel();
21188        let stdout_sink = sink.clone();
21189        let pump = tokio::spawn(async move {
21190            pump_stdout_to(reader, stdout_sink).await;
21191            finished.send(()).unwrap();
21192        });
21193        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
21194
21195        let started = Instant::now();
21196        settle_stdout_pump("two-pipe", pump, BOUND).await;
21197        assert_eq!(
21198            started.elapsed(),
21199            BOUND,
21200            "a late reader must not hold up restart"
21201        );
21202        assert_eq!(
21203            std::fs::read_to_string(&path).unwrap().lines().count(),
21204            BURST + 1
21205        );
21206
21207        release
21208            .send(())
21209            .expect("stdout reader must survive the exit drain bound");
21210        tokio::time::timeout(Duration::from_secs(1), completion)
21211            .await
21212            .expect("released stdout reader must reach EOF")
21213            .expect("stdout capture must finish at EOF, not be cancelled");
21214        let contents = std::fs::read_to_string(&path).unwrap();
21215        let mut lines = contents
21216            .lines()
21217            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
21218            .collect::<Vec<_>>();
21219        let mut expected = (0..BURST)
21220            .flat_map(|i| {
21221                [
21222                    format!("out-{i:04}-{}", "o".repeat(64)),
21223                    format!("err-{i:04}-{}", "e".repeat(64)),
21224                ]
21225            })
21226            .collect::<Vec<_>>();
21227        lines.sort_unstable();
21228        expected.sort_unstable();
21229        assert_eq!(
21230            lines, expected,
21231            "both pipes must retain every complete line exactly once"
21232        );
21233    }
21234}
21235
21236/// Containment of a module's process tree (issue #109).
21237///
21238/// The behaviour these defend against is a module helper surviving its module:
21239/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
21240/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
21241/// compounds it.
21242///
21243/// They run against the SUPERVISOR rather than the job-object crate because the
21244/// claim is about teardown: a crate-level test proves a job can reap a tree, not
21245/// that the daemon's drain path reaches it.
21246///
21247/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
21248/// lane there is a separate containment path with its own tests.
21249#[cfg(all(test, windows))]
21250mod job_containment_tests {
21251    use super::*;
21252    use cortexkit_test_support::ScratchDir;
21253    use std::{
21254        path::{Path, PathBuf},
21255        sync::{Arc, Mutex},
21256        time::{Duration, Instant},
21257    };
21258
21259    /// The stub, expected beside this test executable.
21260    ///
21261    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
21262    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
21263    /// failure then reads as a broken test rather than an unbuilt dependency.
21264    fn stub_path() -> PathBuf {
21265        let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
21266        assert!(
21267            path.exists(),
21268            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
21269             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
21270            path.display()
21271        );
21272        path
21273    }
21274
21275    /// Poll for the grandchild pid the stub records, and parse it.
21276    fn read_grandchild_pid(path: &Path) -> u32 {
21277        let deadline = Instant::now() + Duration::from_secs(10);
21278        loop {
21279            if let Ok(contents) = std::fs::read_to_string(path) {
21280                if let Ok(pid) = contents.trim().parse() {
21281                    return pid;
21282                }
21283            }
21284            assert!(
21285                Instant::now() < deadline,
21286                "the stub never recorded a grandchild pid at {}",
21287                path.display()
21288            );
21289            std::thread::sleep(Duration::from_millis(10));
21290        }
21291    }
21292
21293    /// Everything one fixture run needs, so the two tests below differ in exactly
21294    /// one place: whether the child is contained.
21295    struct Fixture {
21296        _dir: ScratchDir,
21297        _roster: ChildRoster,
21298        module_id: String,
21299        grandchild: u32,
21300        child: Option<SupervisedChild>,
21301        registry: Arc<Registry>,
21302        snapshot: Arc<Mutex<SupervisorSnapshot>>,
21303        terminal_ring: Arc<Mutex<TerminalRing>>,
21304        spawn_events: SpawnEventFeed,
21305    }
21306
21307    fn fixture(label: &str, module_id: &str) -> Fixture {
21308        fixture_with_carry(label, module_id, false)
21309    }
21310
21311    fn fixture_with_carry(label: &str, module_id: &str, carry_containment: bool) -> Fixture {
21312        let dir = ScratchDir::new(label);
21313        let pid_file = dir.join("grandchild.pid");
21314        let supervisor = Supervisor::new_for_test(
21315            Arc::new(Registry::default()),
21316            RestartPolicy::new(3, Duration::ZERO),
21317        );
21318        let runtime = supervisor.runtime_config();
21319        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21320        let spec = ModuleSpec {
21321            script: None,
21322            module_id: module_id.to_string(),
21323            program: stub_path(),
21324            // Zero args deliberately: a `--subc` argument would make the stub dial
21325            // a daemon that is not there, and the failure would land in the same
21326            // stderr ring this fixture exists to keep quiet.
21327            args: Vec::new(),
21328            env: vec![
21329                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
21330                (
21331                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
21332                    pid_file.display().to_string(),
21333                ),
21334            ],
21335            reserved: false,
21336            reserved_prefixes: Vec::new(),
21337            protocol: ModuleProtocol::Subc,
21338            overlap: Default::default(),
21339            carry_containment,
21340        };
21341        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
21342            .expect("spawn the supervised fixture");
21343        let grandchild = read_grandchild_pid(&pid_file);
21344        Fixture {
21345            _dir: dir,
21346            _roster: runtime.child_roster.clone(),
21347            module_id: module_id.to_string(),
21348            grandchild,
21349            child: Some(child),
21350            registry: Arc::new(Registry::default()),
21351            snapshot,
21352            terminal_ring: Arc::clone(&runtime.terminal_ring),
21353            spawn_events: SpawnEventFeed::default(),
21354        }
21355    }
21356
21357    async fn assert_admission_kill_reaps_grandchild(carry: bool) {
21358        let label = if carry {
21359            "admission-kill-carried-tree"
21360        } else {
21361            "admission-kill-plain-tree"
21362        };
21363        let mut fixture = fixture_with_carry(label, label, carry);
21364        let descendant = subc_os::Process::open(fixture.grandchild)
21365            .unwrap()
21366            .expect("the stub descendant must be alive before admission kill");
21367        assert!(descendant.observe().is_some());
21368        let child = fixture.child.as_mut().unwrap();
21369        assert_eq!(child.child.container().is_some(), carry);
21370        assert_eq!(child.job.is_some(), !carry);
21371
21372        child.start_admission_kill().unwrap();
21373        assert!(child.admission_kill_sent);
21374        let status = timeout(Duration::from_secs(10), child.wait())
21375            .await
21376            .unwrap()
21377            .unwrap();
21378        assert_eq!(
21379            status.code(),
21380            Some(admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE),
21381            "containment teardown must preserve the leader's distinctive status"
21382        );
21383        // Keep the child and its job/container alive: later exit finalization or
21384        // handle drop must not be what makes the descendant-death claim pass.
21385        assert!(
21386            descendant.wait_for_exit(Duration::from_secs(10)).unwrap(),
21387            "admission kill left a descendant running before exit finalization"
21388        );
21389        assert_eq!(child.child.container().is_some(), carry);
21390        assert_eq!(child.job.is_some(), !carry);
21391        child.finish_exit(false).await.unwrap();
21392    }
21393
21394    #[tokio::test]
21395    async fn admission_kill_reaps_grandchild_with_plain_job() {
21396        assert_admission_kill_reaps_grandchild(false).await;
21397    }
21398
21399    #[tokio::test]
21400    async fn admission_kill_reaps_grandchild_with_carried_job() {
21401        assert_admission_kill_reaps_grandchild(true).await;
21402    }
21403
21404    #[tokio::test]
21405    async fn admission_kill_retires_registration_before_wait() {
21406        let mut fixture = fixture("admission-registration", "admission-registration");
21407        let handle = SupervisorHandle::new();
21408        let spawn =
21409            RegistrationSpawn::new(&handle, "admission-registration", SpawnRole::Plain, None);
21410        spawn.ready();
21411        let child = fixture.child.as_mut().unwrap();
21412        child.registration_spawn = Some(spawn.clone());
21413        child.start_admission_kill().unwrap();
21414        assert!(
21415            !spawn.launch_ready(),
21416            "admission kill must retire before its caller can await the child"
21417        );
21418        timeout(Duration::from_secs(10), child.wait())
21419            .await
21420            .unwrap()
21421            .unwrap();
21422        child.finish_exit(false).await.unwrap();
21423    }
21424
21425    impl Fixture {
21426        /// Drain through the supervisor's own teardown path.
21427        async fn drain(&mut self) {
21428            let child = self
21429                .child
21430                .take()
21431                .expect("the fixture child is still present");
21432            drain_child_to_state(
21433                &self.module_id,
21434                ModuleProtocol::Subc,
21435                // No forwarding table in this fixture, so nothing reaches the
21436                // child over a connection.
21437                StopNotice::NotSent,
21438                &self.registry,
21439                None,
21440                &self.snapshot,
21441                &self.terminal_ring,
21442                &self.spawn_events,
21443                child,
21444                Duration::from_millis(500),
21445                ModuleState::Stopped,
21446                Some(false),
21447                &FailureContext::operator(),
21448            )
21449            .await
21450            .expect("drain the supervised fixture");
21451        }
21452    }
21453
21454    /// Teardown reaps the grandchild, not merely the direct child.
21455    ///
21456    /// This is the assertion the change exists for. Before containment the
21457    /// grandchild survived: it is a separate process, and `start_kill` is
21458    /// `TerminateProcess` scoped to one pid.
21459    #[tokio::test]
21460    async fn teardown_reaps_the_grandchild() {
21461        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
21462        let grandchild = fixture.grandchild;
21463
21464        assert!(
21465            subc_jobobject::process_exists(grandchild),
21466            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
21467        );
21468
21469        fixture.drain().await;
21470
21471        assert!(
21472            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21473            "grandchild {grandchild} outlived module teardown: the tree was not contained"
21474        );
21475    }
21476
21477    /// The mutation control: with containment withheld, the grandchild survives
21478    /// the same kill.
21479    ///
21480    /// This is the defect reproduction from #109 — a direct-child kill reaches
21481    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
21482    /// supervisor because `spawn_and_mark_running` now always contains on
21483    /// Windows, which is the point: there is no longer a path that spawns
21484    /// uncontained, so the control has to construct one.
21485    ///
21486    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
21487    /// grandchild ever dies here, that test is passing for a reason unrelated to
21488    /// the job object and the containment claim is unproven.
21489    #[test]
21490    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
21491        let dir = ScratchDir::new("teardown-uncontained");
21492        let pid_file = dir.join("grandchild.pid");
21493        let mut child = std::process::Command::new(stub_path())
21494            .env("FAKE_AFT_NEVER_CONNECT", "1")
21495            .env(
21496                "FAKE_AFT_GRANDCHILD_PID_FILE",
21497                pid_file.display().to_string(),
21498            )
21499            .stdin(std::process::Stdio::null())
21500            .stdout(std::process::Stdio::null())
21501            .stderr(std::process::Stdio::null())
21502            .spawn()
21503            .expect("spawn the uncontained fixture");
21504        let grandchild = read_grandchild_pid(&pid_file);
21505
21506        // Exactly what the pre-fix teardown did: kill the direct child.
21507        child.kill().expect("kill the direct child");
21508        let _ = child.wait();
21509
21510        assert!(
21511            subc_jobobject::process_exists(grandchild),
21512            "grandchild {grandchild} died with the direct child, so this control no longer \
21513             distinguishes contained from uncontained teardown and the regression test is \
21514             passing vacuously"
21515        );
21516
21517        // The orphan this control demonstrates is the leak the fix prevents, so
21518        // the control must not leave one behind.
21519        kill_tree(grandchild);
21520    }
21521
21522    /// Crash durability: closing the containment handle reaps the tree with no
21523    /// teardown code running at all.
21524    ///
21525    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
21526    /// call anything — and it is why containment is a kernel property of the
21527    /// handle rather than a step in the drain. Discovered by getting the
21528    /// mutation control wrong: clearing `job` to "disable" containment instead
21529    /// killed the tree, which is the guarantee, not a mistake.
21530    #[tokio::test]
21531    async fn dropping_containment_reaps_the_grandchild() {
21532        let mut fixture = fixture("drop-containment", "tree-drop");
21533        let grandchild = fixture.grandchild;
21534
21535        assert!(subc_jobobject::process_exists(grandchild));
21536
21537        // No `drain` call, no kill: dropping the handle is the entire mechanism.
21538        fixture.child.as_mut().expect("child present").job = None;
21539
21540        assert!(
21541            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21542            "grandchild {grandchild} survived the containment handle closing, so a daemon \
21543             crash would leave the tree behind"
21544        );
21545    }
21546
21547    /// Kill a pid and its tree, then confirm it is gone.
21548    fn kill_tree(pid: u32) {
21549        let _ = std::process::Command::new("taskkill.exe")
21550            .args(["/PID", &pid.to_string(), "/T", "/F"])
21551            .stdin(std::process::Stdio::null())
21552            .stdout(std::process::Stdio::null())
21553            .stderr(std::process::Stdio::null())
21554            .status();
21555        assert!(
21556            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
21557            "could not clean up grandchild {pid}"
21558        );
21559    }
21560}
21561
21562#[cfg(test)]
21563mod privacy_trampoline_configuration_tests {
21564    #[tokio::test]
21565    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21566    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
21567        #[cfg(target_os = "macos")]
21568        {
21569            let supervisor = super::Supervisor::new(
21570                std::sync::Arc::new(crate::Registry::default()),
21571                super::RestartPolicy::default(),
21572            );
21573            let error = supervisor.spawn(spec()).unwrap_err();
21574            assert!(
21575                error
21576                    .to_string()
21577                    .contains("no privacy trampoline configured"),
21578                "{error}"
21579            );
21580        }
21581    }
21582
21583    #[tokio::test]
21584    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21585    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
21586        #[cfg(target_os = "macos")]
21587        {
21588            let supervisor = super::Supervisor::new(
21589                std::sync::Arc::new(crate::Registry::default()),
21590                super::RestartPolicy::default(),
21591            )
21592            .with_privacy_trampoline(std::env::current_exe().unwrap());
21593            let error = supervisor.spawn(spec()).unwrap_err();
21594            assert!(
21595                error
21596                    .to_string()
21597                    .contains("binary does not implement the privacy trampoline protocol"),
21598                "{error}"
21599            );
21600        }
21601    }
21602
21603    #[cfg(target_os = "macos")]
21604    fn spec() -> super::ModuleSpec {
21605        super::ModuleSpec {
21606            module_id: "privacy-configuration".into(),
21607            program: "/bin/sleep".into(),
21608            args: vec!["30".into()],
21609            env: vec![],
21610            reserved: false,
21611            reserved_prefixes: vec![],
21612            protocol: subc_control::ModuleProtocol::None,
21613            overlap: super::ModuleOverlap::Exclusive,
21614            carry_containment: false,
21615            script: None,
21616        }
21617    }
21618}
21619
21620#[cfg(test)]
21621mod privacy_exec_boundary_tests {
21622    #[cfg(target_os = "macos")]
21623    use super::*;
21624    #[cfg(target_os = "macos")]
21625    use std::{
21626        io::{Read, Write},
21627        net::{TcpListener, TcpStream},
21628    };
21629
21630    /// Unit-test-only pause at the actual early image read, not at a later
21631    /// status read. Production supervisors never inspect this environment key.
21632    #[cfg(target_os = "macos")]
21633    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
21634        if let Some((_, path)) = spec
21635            .env
21636            .iter()
21637            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
21638        {
21639            let mut barrier = TcpStream::connect(path).unwrap();
21640            barrier
21641                .set_read_timeout(Some(Duration::from_secs(30)))
21642                .unwrap();
21643            barrier.write_all(&pid.to_ne_bytes()).unwrap();
21644            let mut release = [0];
21645            barrier.read_exact(&mut release).unwrap();
21646            assert_eq!(&release, b"X");
21647        }
21648    }
21649
21650    #[cfg(target_os = "macos")]
21651    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
21652        ModuleSpec {
21653            script: None,
21654            module_id: "privacy-boundary".into(),
21655            program: program.into(),
21656            args: args.iter().map(|arg| (*arg).into()).collect(),
21657            env: vec![],
21658            reserved: false,
21659            reserved_prefixes: vec![],
21660            protocol: ModuleProtocol::None,
21661            overlap: ModuleOverlap::Exclusive,
21662            carry_containment: false,
21663        }
21664    }
21665
21666    #[cfg(target_os = "macos")]
21667    async fn accept(listener: TcpListener) -> TcpStream {
21668        // Socket readiness, not elapsed time, establishes both pause points.
21669        let listener = tokio::net::TcpListener::from_std({
21670            listener.set_nonblocking(true).unwrap();
21671            listener
21672        })
21673        .unwrap();
21674        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
21675            .await
21676            .unwrap()
21677            .unwrap();
21678        let stream = stream.into_std().unwrap();
21679        stream.set_nonblocking(false).unwrap();
21680        stream
21681            .set_read_timeout(Some(Duration::from_secs(30)))
21682            .unwrap();
21683        stream
21684    }
21685
21686    #[tokio::test]
21687    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21688    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
21689        #[cfg(target_os = "macos")]
21690        {
21691            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
21692            // Loopback sockets also work when the replay adapter's TMPDIR is
21693            // longer than Darwin's Unix-domain socket path limit.
21694            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21695            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21696            let record = root.join("live-children.json");
21697            let supervisor =
21698                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21699                    .with_live_children_record(&record);
21700            let runtime = supervisor.runtime_config();
21701            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21702            let mut spec = spec("/bin/sleep", &["30"]);
21703            spec.env = vec![
21704                (
21705                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21706                    exec_listener.local_addr().unwrap().to_string(),
21707                ),
21708                (
21709                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
21710                    sample_listener.local_addr().unwrap().to_string(),
21711                ),
21712            ];
21713            let spawn = tokio::task::spawn_blocking(move || {
21714                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
21715            });
21716            let mut sample = accept(sample_listener).await;
21717            let mut pid = [0; 4];
21718            sample.read_exact(&mut pid).unwrap();
21719            let pid = u32::from_ne_bytes(pid);
21720            let mut exec = accept(exec_listener).await;
21721            let mut ready = [0];
21722            exec.read_exact(&mut ready).unwrap();
21723            assert_eq!(&ready, b"R");
21724            // The early read is guaranteed to see a real, non-null trampoline
21725            // image: the fixture has reached its barrier and cannot exec yet.
21726            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
21727            assert_eq!(
21728                observe_spawned_image(pid).unwrap().executable,
21729                Some(trampoline)
21730            );
21731            sample.write_all(b"X").unwrap();
21732            let mut child = spawn.await.unwrap();
21733            let early = crate::live_children::read_record(&record).unwrap();
21734            assert_eq!(early.len(), 1);
21735            assert_eq!(early[0].pid, pid);
21736            assert_eq!(
21737                early[0].executable, None,
21738                "unconfirmed trampoline image entered the roster"
21739            );
21740            assert!(child.report_ready.get().is_none());
21741            // The barrier's duration is unrelated to the production five-second
21742            // exec budget. Start the test's confirmation budget upon release.
21743            child.privacy_exec.as_mut().unwrap().deadline =
21744                tokio::time::Instant::now() + Duration::from_secs(30);
21745            exec.write_all(b"X").unwrap();
21746            child.confirm_privacy_exec().await;
21747            assert_eq!(child.spawn_failure, None);
21748            assert!(child.report_ready.get().is_some());
21749            let confirmed = crate::live_children::read_record(&record).unwrap();
21750            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
21751            assert_ne!(module, trampoline);
21752            assert_eq!(confirmed[0].executable, Some(module.into()));
21753            child.start_kill().unwrap();
21754            child.wait().await.unwrap();
21755            child.release_roster();
21756        }
21757    }
21758
21759    #[tokio::test]
21760    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21761    async fn macos_unstamped_reexec_records_the_observed_native_image() {
21762        #[cfg(target_os = "macos")]
21763        {
21764            let root = cortexkit_test_support::ScratchDir::new("privacy-reexec");
21765            let record = root.join("live-children.json");
21766            let supervisor =
21767                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21768                    .with_live_children_record(&record);
21769            let runtime = supervisor.runtime_config();
21770            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21771            let fixture = crate::config_gate::program::native_fixture_for_test();
21772            let mut spec = spec(
21773                "/bin/sh",
21774                &["-c", "exec \"$1\" 600", "reexec", fixture.to_str().unwrap()],
21775            );
21776            spec.env
21777                .push(("SUBC_TEST_PRIVACY_IMAGE_DELAY_MS".into(), "5".into()));
21778            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21779            let native_image = subc_os::file_identity(&fixture).unwrap();
21780            // Native image installation, not shell startup speed, establishes
21781            // that this launch has crossed the re-exec boundary before sampling.
21782            let reexec = tokio::time::timeout(Duration::from_secs(5), async {
21783                while observe_spawned_image(child.pid).and_then(|image| image.executable)
21784                    != Some(native_image)
21785                {
21786                    tokio::time::sleep(Duration::from_millis(1)).await;
21787                }
21788            })
21789            .await;
21790            child.confirm_privacy_exec().await;
21791            let failure = child.spawn_failure.clone();
21792            let ready = child.report_ready.get().is_some();
21793            let confirmed = crate::live_children::read_record(&record).unwrap();
21794            child.start_kill().unwrap();
21795            tokio::time::timeout(Duration::from_secs(5), child.wait())
21796                .await
21797                .unwrap()
21798                .unwrap();
21799            child.release_roster();
21800
21801            reexec.expect("the shell must re-exec the native fixture");
21802            assert_eq!(
21803                failure, None,
21804                "an unstamped program may re-exec before observation"
21805            );
21806            assert!(ready);
21807            assert_eq!(confirmed.len(), 1);
21808            assert_eq!(confirmed[0].executable, Some(native_image.into()));
21809        }
21810    }
21811
21812    #[tokio::test]
21813    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21814    async fn macos_unstamped_ack_without_exec_still_refuses_the_trampoline() {
21815        #[cfg(target_os = "macos")]
21816        {
21817            let listener = TcpListener::bind("127.0.0.1:0").unwrap();
21818            let supervisor =
21819                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
21820            let runtime = supervisor.runtime_config();
21821            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21822            let fixture = crate::config_gate::program::native_fixture_for_test();
21823            let mut spec = spec(fixture.to_str().unwrap(), &["600"]);
21824            spec.env.push((
21825                "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21826                listener.local_addr().unwrap().to_string(),
21827            ));
21828            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21829            let mut exec = accept(listener).await;
21830            let mut ready = [0];
21831            exec.read_exact(&mut ready).unwrap();
21832            assert_eq!(&ready, b"R");
21833            assert_eq!(
21834                observe_spawned_image(child.pid).unwrap().executable,
21835                subc_os::file_identity(&test_privacy_trampoline())
21836            );
21837            // Simulate a broken trampoline that acknowledges exec but keeps
21838            // running its own image, held at the fixture's pre-exec barrier.
21839            let (reader, writer) = std::io::pipe().unwrap();
21840            drop(writer);
21841            child.privacy_exec.as_mut().unwrap().reader =
21842                tokio::io::unix::AsyncFd::new(reader).unwrap();
21843            child.confirm_privacy_exec().await;
21844            let failure = child.spawn_failure.clone();
21845            let ready = child.report_ready.get().is_some();
21846            child.start_kill().unwrap();
21847            tokio::time::timeout(Duration::from_secs(5), child.wait())
21848                .await
21849                .unwrap()
21850                .unwrap();
21851            child.release_roster();
21852            assert_eq!(
21853                failure.as_deref(),
21854                Some("privacy identity trampoline did not exec the module")
21855            );
21856            assert!(!ready);
21857        }
21858    }
21859
21860    #[tokio::test]
21861    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21862    async fn macos_resumed_121_is_a_module_exit_not_a_trampoline_refusal() {
21863        #[cfg(target_os = "macos")]
21864        {
21865            let registry = Arc::new(Registry::default());
21866            let policy = RestartPolicy::new(0, Duration::ZERO);
21867            let fixture = crate::config_gate::program::native_fixture_for_test();
21868            let spec = spec(fixture.to_str().unwrap(), &["exit121"]);
21869            let scratch = cortexkit_test_support::ScratchDir::new("resumed-exit");
21870            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy)
21871                .with_config_runtime(suspended_spawn_tests::gated_owner(&scratch, &spec, true));
21872            let runtime = supervisor.runtime_config();
21873            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21874            // A module cannot exit before suspended admission. After resume,
21875            // retain the real exit status even when it resembles a refusal.
21876            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21877            child.confirm_privacy_exec().await;
21878            assert_eq!(child.spawn_failure, None);
21879            assert!(child.report_ready.get().is_some());
21880            let pid = child.pid;
21881            tokio::task::spawn_blocking(move || {
21882                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
21883            })
21884            .await
21885            .unwrap()
21886            .unwrap();
21887            let status = child.wait().await.unwrap();
21888            assert_eq!(status.code(), Some(121));
21889            assert!(child.privacy_exec.is_none());
21890            let report = classify_reaped_child_exit(&snapshot, &child, &status);
21891            on_child_exit(
21892                &spec,
21893                policy,
21894                &registry,
21895                &snapshot,
21896                &runtime.terminal_ring,
21897                &runtime.spawn_events,
21898                &runtime.child_roster,
21899                report,
21900                true,
21901            )
21902            .await;
21903            let state = lock_snapshot(&snapshot).unwrap();
21904            assert_eq!(state.state, ModuleState::Failed);
21905            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
21906            assert_eq!(state.reported_pid(), None);
21907            drop(state);
21908            let history = runtime.terminal_ring.lock().unwrap().snapshot();
21909            assert_eq!(history.entries.len(), 1);
21910            let terminal = &history.entries[0];
21911            assert_eq!(terminal.exit_code, Some(121));
21912            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
21913            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
21914            assert_eq!(
21915                terminal.disposition_detail.as_deref(),
21916                Some(policy.budget_exhausted_detail().as_str()),
21917                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
21918            );
21919            assert_eq!(child.spawn_failure, None);
21920            child.release_roster();
21921        }
21922    }
21923}
21924
21925#[cfg(all(test, target_os = "macos"))]
21926#[path = "supervise/suspended_spawn_tests.rs"]
21927mod suspended_spawn_tests;
21928
21929/// The daemon's real spawn path hands a subc-wire child its launch nonce on
21930/// descriptor 3, without an environment copy. The shell records the nonce
21931/// and its environment after exec so these tests observe the real handover.
21932#[cfg(all(test, unix))]
21933mod launch_nonce_descriptor_tests {
21934    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
21935    use crate::stderr_tail::{StderrRing, StderrTailConfig};
21936    use cortexkit_test_support::ScratchDir;
21937    use std::{
21938        path::PathBuf,
21939        sync::{Arc, Mutex},
21940        time::{Duration, Instant},
21941    };
21942
21943    async fn probe(role: super::SpawnRole) {
21944        let scratch = ScratchDir::new("launch-nonce-descriptor");
21945        let fd_copy = scratch.join("from-descriptor");
21946        let env_copy = scratch.join("environment");
21947        let script = format!(
21948            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
21949            fd = fd_copy.display(), env = env_copy.display(),
21950        );
21951        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
21952        let spec = ModuleSpec {
21953            script: None,
21954            module_id: "nonce-descriptor-probe".to_string(),
21955            program: PathBuf::from("/bin/sh"),
21956            args: vec!["-c".to_string(), script],
21957            env: vec![
21958                xdg("XDG_DATA_HOME"),
21959                xdg("XDG_RUNTIME_DIR"),
21960                xdg("XDG_CONFIG_HOME"),
21961                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
21962            ],
21963            reserved: true,
21964            reserved_prefixes: Vec::new(),
21965            protocol: ModuleProtocol::Subc,
21966            overlap: Default::default(),
21967            carry_containment: false,
21968        };
21969        let handle = SupervisorHandle::new();
21970        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21971        let roster = ChildRoster::default();
21972        #[cfg(target_os = "macos")]
21973        {
21974            let path = super::test_privacy_trampoline();
21975            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
21976        }
21977        let child = super::spawn_child_in_slot(
21978            &spec,
21979            None,
21980            Some(&handle),
21981            &ring,
21982            None,
21983            &roster,
21984            None,
21985            #[cfg(target_os = "linux")]
21986            None,
21987            role,
21988            matches!(role, super::SpawnRole::SwapCandidate),
21989            super::DEFAULT_REGISTER_TIMEOUT,
21990        )
21991        .expect("spawn probe");
21992        let deadline = Instant::now() + Duration::from_secs(10);
21993        while !(fd_copy.exists() && env_copy.exists()) {
21994            assert!(Instant::now() < deadline, "probe never wrote its copies");
21995            tokio::time::sleep(Duration::from_millis(20)).await;
21996        }
21997        let nonce = std::fs::read_to_string(fd_copy).unwrap();
21998        assert!(!nonce.is_empty());
21999        let environment = std::fs::read_to_string(env_copy).unwrap();
22000        assert!(environment
22001            .lines()
22002            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
22003        let copy = environment
22004            .lines()
22005            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
22006        assert_eq!(
22007            copy, None,
22008            "Unix children must never receive the environment nonce"
22009        );
22010        if matches!(role, super::SpawnRole::Plain) {
22011            assert_eq!(
22012                handle.spawn_nonce(&spec.module_id).as_deref(),
22013                Some(nonce.as_str())
22014            );
22015        }
22016        drop(child);
22017    }
22018
22019    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
22020    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
22021        probe(super::SpawnRole::Plain).await;
22022    }
22023
22024    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
22025    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
22026        probe(super::SpawnRole::SwapCandidate).await;
22027    }
22028}
22029
22030#[cfg(all(test, target_os = "linux"))]
22031#[path = "supervise/cgroup_reap_tests.rs"]
22032mod cgroup_reap_tests;
22033
22034#[cfg(all(test, target_os = "linux"))]
22035mod cgroup_containment_tests {
22036    use super::*;
22037    use cortexkit_test_support::ScratchDir;
22038
22039    #[tokio::test]
22040    async fn linux_subtree_memory_covers_module_children() {
22041        let dir = ScratchDir::new("subtree-memory");
22042        let root = PathBuf::from(format!(
22043            "/sys/fs/cgroup/subc-memory-test-{}-{}",
22044            std::process::id(),
22045            unix_ms_now()
22046        ));
22047        if let Err(error) = std::fs::create_dir(&root) {
22048            assert!(
22049                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
22050                "required cgroup test cannot execute: {error}"
22051            );
22052            eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
22053            return;
22054        }
22055        let placement = subc_cgroup::prepare_at(&root)
22056            .unwrap()
22057            .expect("delegated cgroup");
22058        let supervisor = Supervisor::new_for_test(
22059            Arc::new(Registry::default()),
22060            RestartPolicy::new(3, Duration::ZERO),
22061        )
22062        .with_cgroup_placement(Some(placement));
22063        let ready = dir.join("ready");
22064        let module = supervisor
22065            .spawn(ModuleSpec {
22066                script: None,
22067                module_id: "memory-tree".into(),
22068                program: PathBuf::from("/bin/sh"),
22069                args: vec![
22070                    "-c".into(),
22071                    "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
22072                    "fixture".into(),
22073                    ready.display().to_string(),
22074                ],
22075                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
22076                    .into_iter()
22077                    .map(|key| (key.into(), dir.display().to_string()))
22078                    .collect(),
22079                reserved: false,
22080                reserved_prefixes: Vec::new(),
22081                protocol: ModuleProtocol::None,
22082                overlap: Default::default(),
22083                carry_containment: false,
22084            })
22085            .unwrap();
22086        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
22087        while !ready.exists() {
22088            assert!(
22089                tokio::time::Instant::now() < deadline,
22090                "children never started"
22091            );
22092            tokio::time::sleep(Duration::from_millis(10)).await;
22093        }
22094        let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
22095        let reading = &readings[0];
22096        assert!(
22097            reading.process_count.is_some_and(|count| count >= 4),
22098            "{reading:?}"
22099        );
22100        assert!(
22101            reading.memory_bytes.is_some_and(|bytes| bytes > 0),
22102            "{reading:?}"
22103        );
22104        module.stop().await.unwrap();
22105        drop(module);
22106        std::fs::remove_dir(root.join("subc-modules")).unwrap();
22107        std::fs::remove_dir(&root).unwrap();
22108        eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
22109    }
22110
22111    fn running(pid: u32) -> bool {
22112        // An orphan can remain a zombie until the container init reaps it.
22113        std::fs::read_to_string(format!("/proc/{pid}/stat"))
22114            .ok()
22115            .and_then(|stat| {
22116                stat.rsplit_once(") ")
22117                    .map(|(_, rest)| rest.starts_with('Z'))
22118            })
22119            .is_some_and(|zombie| !zombie)
22120    }
22121
22122    #[tokio::test]
22123    async fn linux_teardown_reaps_the_grandchild() {
22124        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
22125    }
22126
22127    #[tokio::test]
22128    async fn linux_shutdown_straggler_reaps_the_grandchild() {
22129        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
22130    }
22131
22132    async fn teardown_tree(test_name: &str, shutdown: bool) {
22133        let dir = ScratchDir::new(test_name);
22134        let root = PathBuf::from(format!(
22135            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
22136            std::process::id(),
22137            unix_ms_now()
22138        ));
22139        if let Err(error) = std::fs::create_dir(&root) {
22140            assert!(
22141                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
22142                "required cgroup test cannot execute: {error}"
22143            );
22144            eprintln!(
22145                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
22146                root.display()
22147            );
22148            return;
22149        }
22150        let placement = subc_cgroup::prepare_at(&root)
22151            .expect("prepare isolated kernel cgroup")
22152            .expect("isolated cgroup is delegated");
22153        let module_id = "tree-teardown";
22154        let module = placement
22155            .module_path(module_id)
22156            .expect("create isolated module cgroup");
22157        if !module.join("cgroup.kill").exists() {
22158            std::fs::remove_dir(&module).unwrap();
22159            std::fs::remove_dir(root.join("subc-modules")).unwrap();
22160            std::fs::remove_dir(&root).unwrap();
22161            assert!(
22162                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
22163                "required cgroup.kill interface unavailable"
22164            );
22165            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
22166            return;
22167        }
22168        let supervisor = Supervisor::new_for_test(
22169            Arc::new(Registry::default()),
22170            RestartPolicy::new(3, Duration::ZERO),
22171        )
22172        .with_cgroup_placement(Some(placement));
22173        let mut runtime = supervisor.runtime_config();
22174        runtime.child_roster = runtime
22175            .child_roster
22176            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
22177        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
22178        let pid_file = dir.join("grandchild.pid");
22179        let spec = ModuleSpec {
22180            script: None,
22181            module_id: module_id.to_string(),
22182            program: PathBuf::from("/bin/sh"),
22183            args: vec![
22184                "-c".into(),
22185                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
22186                "fixture".into(),
22187                pid_file.display().to_string(),
22188            ],
22189            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
22190                .into_iter()
22191                .map(|key| (key.to_string(), dir.display().to_string()))
22192                .collect(),
22193            reserved: false,
22194            reserved_prefixes: Vec::new(),
22195            protocol: ModuleProtocol::None,
22196            overlap: Default::default(),
22197            carry_containment: false,
22198        };
22199        let child =
22200            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
22201        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
22202        let grandchild: u32 = loop {
22203            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
22204                if let Ok(pid) = contents.trim().parse() {
22205                    break pid;
22206                }
22207            }
22208            assert!(
22209                tokio::time::Instant::now() < deadline,
22210                "grandchild pid was not recorded"
22211            );
22212            tokio::time::sleep(Duration::from_millis(10)).await;
22213        };
22214        assert!(
22215            running(grandchild),
22216            "grandchild must be alive before teardown"
22217        );
22218        if shutdown {
22219            let mut child = child;
22220            crate::child_roster::end_children_for_daemon_shutdown(
22221                &runtime.child_roster,
22222                false,
22223                std::future::pending(),
22224            )
22225            .await;
22226            child.wait().await.expect("reap shutdown straggler");
22227        } else {
22228            drain_child_to_state(
22229                module_id,
22230                ModuleProtocol::None,
22231                StopNotice::NotSent,
22232                &Registry::default(),
22233                None,
22234                &snapshot,
22235                &runtime.terminal_ring,
22236                &SpawnEventFeed::default(),
22237                child,
22238                Duration::from_millis(100),
22239                ModuleState::Stopped,
22240                Some(false),
22241                &FailureContext::operator(),
22242            )
22243            .await
22244            .expect("real supervisor teardown");
22245        }
22246        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
22247        while running(grandchild) && tokio::time::Instant::now() < deadline {
22248            tokio::time::sleep(Duration::from_millis(10)).await;
22249        }
22250        let survived = running(grandchild);
22251        // Kill a surviving grandchild so a failed test does not leave it behind.
22252        if survived {
22253            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
22254            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
22255            tokio::time::sleep(Duration::from_millis(100)).await;
22256        }
22257        if module.exists() {
22258            std::fs::remove_dir(&module).expect("remove empty module cgroup");
22259        }
22260        std::fs::remove_dir(root.join("subc-modules")).unwrap();
22261        std::fs::remove_dir(&root).unwrap();
22262        assert!(
22263            !survived,
22264            "grandchild {grandchild} outlived module teardown"
22265        );
22266        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
22267    }
22268}
22269
22270/// A slow live-children record write on the restart path must not stop the
22271/// daemon from answering the replacement module's authentication handshake.
22272#[cfg(all(test, unix))]
22273mod record_write_stall_tests {
22274    use super::*;
22275    use std::sync::atomic::{AtomicBool, Ordering};
22276    use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
22277
22278    /// `subc-client-rs` gives the whole handshake this long before the module
22279    /// exits with `Timeout { stage: ServerProof }`.
22280    const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
22281    /// How long the replacement's admission write is held: longer than the
22282    /// module's deadline, standing in for the delay a busy disk can add
22283    /// across the two syncs in that write.
22284    const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
22285
22286    /// A module that runs `/bin/sleep` directly, not through `/bin/sh -c`.
22287    /// On macOS every module is launched through the privacy trampoline, and
22288    /// a module that is not a `#!` script is admitted only if its running
22289    /// image is the configured program. With `/bin/sh -c "exec sleep 600"`
22290    /// the shell soon replaces itself with `/bin/sleep`, which does not match
22291    /// the configured `/bin/sh`. The image is read only after the spawn's
22292    /// live-children record write, and here that write lasted long enough for
22293    /// the shell to have exec'd every time, so each process was refused and
22294    /// killed right after spawn, and the supervisor crash-looped it with
22295    /// backoff until its restart budget ran out. The restart this test
22296    /// measures would then never produce a module at all.
22297    fn sleeping_spec() -> ModuleSpec {
22298        ModuleSpec {
22299            script: None,
22300            module_id: "record-write-stall".into(),
22301            program: "/bin/sleep".into(),
22302            args: vec!["600".into()],
22303            env: vec![],
22304            reserved: false,
22305            reserved_prefixes: vec![],
22306            protocol: ModuleProtocol::None,
22307            overlap: Default::default(),
22308            carry_containment: false,
22309        }
22310    }
22311
22312    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
22313    async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
22314        let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
22315        let supervisor =
22316            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
22317                .with_live_children_record(home.join("live-children.json"));
22318        let module = supervisor.spawn(sleeping_spec()).unwrap();
22319        // `reported_pid` names a process only once it is confirmed as the
22320        // module (on macOS, once the trampoline's image check has passed), so
22321        // a process the daemon is about to refuse and kill never counts as
22322        // the running module, here or in the final wait below.
22323        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
22324            loop {
22325                if let Some(pid) = lock_snapshot(&module.inner.snapshot)
22326                    .unwrap()
22327                    .reported_pid()
22328                {
22329                    return pid;
22330                }
22331                tokio::time::sleep(Duration::from_millis(10)).await;
22332            }
22333        })
22334        .await
22335        .expect("the first process was admitted as the module");
22336
22337        // The daemon's listener, served by this runtime's workers like the
22338        // supervisor's own tasks.
22339        let key = vec![0x42; 32];
22340        let daemon_id = [0x24; 16];
22341        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22342        let port = listener.local_addr().unwrap().port();
22343        let server = tokio::spawn(crate::server::serve_listener(
22344            listener,
22345            Arc::new(crate::router::Router::with_default_self_handler()),
22346            crate::server::ServerAuth::with_limits(
22347                key.clone(),
22348                daemon_id,
22349                "test",
22350                crate::server::DEFAULT_AUTH_DEADLINE,
22351                4,
22352            ),
22353        ));
22354
22355        // Hold the write that admits the replacement, once, and say when it
22356        // has started.
22357        let (held_tx, held_rx) = std::sync::mpsc::channel();
22358        let armed = AtomicBool::new(true);
22359        supervisor
22360            .child_roster
22361            .set_record_write_hook_for_test(move |live| {
22362                let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
22363                if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
22364                    let _ = held_tx.send(());
22365                    std::thread::sleep(HELD_RECORD_WRITE);
22366                }
22367            });
22368
22369        // The replacement module, as a separate process would be: its own
22370        // thread and runtime, so only the daemon's runtime can be stalled.
22371        let module_process = std::thread::spawn(move || {
22372            held_rx
22373                .recv_timeout(Duration::from_secs(30))
22374                .expect("the restart reached the replacement's record write");
22375            let conn = ConnectionInfo {
22376                schema: SCHEMA_VERSION,
22377                wire_version: None,
22378                endpoints: vec![Endpoint {
22379                    host: "127.0.0.1".to_owned(),
22380                    port,
22381                }],
22382                key,
22383                daemon_id,
22384                pid: std::process::id(),
22385                daemon_ver: "test".to_owned(),
22386            };
22387            tokio::runtime::Builder::new_current_thread()
22388                .enable_all()
22389                .build()
22390                .unwrap()
22391                .block_on(async {
22392                    let started = Instant::now();
22393                    let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
22394                        .await
22395                        .unwrap();
22396                    let result =
22397                        authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
22398                    (result, started.elapsed())
22399                })
22400        });
22401
22402        module.restart(Some(0)).await.unwrap();
22403        let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
22404            .await
22405            .unwrap();
22406        assert!(
22407            auth.is_ok(),
22408            "the daemon must answer a module's handshake while a restart waits on the \
22409             live-children record write; got {auth:?} after {elapsed:?}"
22410        );
22411
22412        tokio::time::timeout(Duration::from_secs(20), async {
22413            while lock_snapshot(&module.inner.snapshot)
22414                .unwrap()
22415                .reported_pid()
22416                .is_none_or(|pid| pid == old_pid)
22417            {
22418                tokio::time::sleep(Duration::from_millis(10)).await;
22419            }
22420        })
22421        .await
22422        .expect("the restart published its replacement");
22423        module.stop().await.unwrap();
22424        server.abort();
22425    }
22426}
22427
22428#[cfg(all(test, unix))]
22429mod retained_status_tests {
22430    use super::*;
22431
22432    #[tokio::test]
22433    async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
22434        use crate::live_children::{ContainerIdentity, RetainedReason};
22435        for (reason, carried, expected) in [
22436            (
22437                RetainedReason::ContainerSurvivors(vec![4242]),
22438                true,
22439                Some("waiting: container survivors (pids [4242])".to_owned()),
22440            ),
22441            (
22442                RetainedReason::ContainerUnverifiable("member inspection denied".into()),
22443                true,
22444                Some(
22445                    "waiting: container survivors could not be verified: member inspection denied"
22446                        .to_owned(),
22447                ),
22448            ),
22449            (RetainedReason::Survived, false, None),
22450        ] {
22451            let supervisor =
22452                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
22453            let spec = ModuleSpec {
22454                script: None,
22455                module_id: "retained-status-fixture".into(),
22456                program: PathBuf::from("/not-spawned"),
22457                args: Vec::new(),
22458                env: Vec::new(),
22459                reserved: false,
22460                reserved_prefixes: Vec::new(),
22461                protocol: ModuleProtocol::None,
22462                overlap: ModuleOverlap::Exclusive,
22463                carry_containment: carried,
22464            };
22465            let module = supervisor
22466                .supervise_configured(spec.clone(), false)
22467                .unwrap();
22468            // No process is launched. Invalid group zero is deliberate: if a
22469            // status read wrongly rechecks, verification changes the reason
22470            // and fails the assertions without signalling any real process.
22471            let guard = supervisor
22472                .child_roster
22473                .admit(
22474                    spec.module_id,
22475                    0,
22476                    ModuleProtocol::None,
22477                    None,
22478                    crate::child_roster::RecordedIdentity {
22479                        container: carried.then_some(ContainerIdentity {
22480                            pgid: Some(0),
22481                            created_start_time: Some(0),
22482                            cgroup_path: None,
22483                            ..ContainerIdentity::test_metadata()
22484                        }),
22485                        ..crate::child_roster::RecordedIdentity::default()
22486                    },
22487                )
22488                .unwrap();
22489            guard.retain(reason);
22490            let status = module.status().unwrap();
22491            assert_eq!(status.waiting_for_previous_detail, expected.clone());
22492            assert_eq!(status.health.detail, expected);
22493            assert!(!status.process_alive);
22494        }
22495    }
22496}
22497
22498fn registration_child_is_running(child: &mut SupervisedChild) -> bool {
22499    child.try_wait().is_ok_and(|status| status.is_none())
22500}