Skip to main content

subc_daemon/
supervise.rs

1//! Identity policy updates read the running-protocol snapshot before taking the
2//! nonce locks. Nested identity locks are taken in this order: spawn nonces,
3//! reserved nonces, then `SupervisorHandle::implicit_protocol_none`.
4
5#[cfg(all(test, target_os = "linux"))]
6#[path = "supervise/audit_identity_tests.rs"]
7mod audit_identity_tests;
8mod resource_launch;
9pub(crate) use resource_launch::{ResourceCommitError, ResourceLaunch};
10
11use std::{
12    collections::{HashMap, HashSet, VecDeque},
13    error::Error,
14    fmt, io,
15    path::PathBuf,
16    process::{ExitStatus, Stdio},
17    sync::{Arc, Mutex, OnceLock},
18    time::{Duration, SystemTime, UNIX_EPOCH},
19};
20
21use cortexkit_log::Retention;
22use serde_json::Value;
23use sha2::{Digest, Sha256};
24use subc_control::{
25    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
26    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
27};
28use subc_protocol::{
29    manifest::{SelfSignalKind, SignalAnchor},
30    session::{
31        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
32        MODULE_CONTROL_OP_HEALTH_CHECK,
33    },
34    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
35};
36use tokio::{
37    process::{Child, Command},
38    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
39    task::JoinHandle,
40    time::{sleep, sleep_until, timeout, timeout_at, Instant},
41};
42use tracing::{debug, error, info, warn};
43
44use crate::{
45    child_roster::ChildRoster,
46    daemon_config::{
47        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
48    },
49    forwarding::{
50        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
51        ModuleDrainTarget, PendingModuleControlRpc,
52    },
53    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
54    registry::{ConnectionId, RegistrationEndReason, RegistryError},
55    stderr_tail::{
56        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
57        StderrTailSnapshot,
58    },
59    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
60    Frame, FrameSink, Registry,
61};
62
63#[cfg(not(windows))]
64use crate::provenance::spawned_file_identity;
65
66#[path = "supervise_swap.rs"]
67mod swap;
68
69#[path = "supervise_activity.rs"]
70mod activity;
71
72#[cfg(unix)]
73#[path = "supervise/admission.rs"]
74mod admission;
75mod admission_exit;
76#[cfg(all(test, target_os = "linux"))]
77mod async_spawn_tests;
78#[cfg(all(test, windows))]
79mod carried_job_tests;
80mod config_delivery;
81mod content_binding;
82mod program_binding;
83mod registration;
84pub(crate) mod script;
85/// Each launch nonce, mapped to its launch's module id and the launch.
86type NonceLaunches = HashMap<String, (String, std::sync::Weak<RegistrationSpawn>)>;
87pub(crate) use registration::NonceAttestation;
88#[cfg(all(test, unix))]
89pub(crate) use registration::RegistrationProof;
90pub(crate) use registration::RegistrationSpawn;
91pub(crate) use registration::REGISTRATION_CHANNEL_CLOSED;
92#[cfg(all(test, unix))]
93pub(crate) mod unix_channel_placement_tests;
94use program_binding::ProgramBindingPublication;
95mod config_exit_failure;
96#[cfg(all(test, any(target_os = "linux", windows)))]
97mod missing_path_tests;
98#[cfg(all(test, unix))]
99mod program_binding_tests;
100#[cfg(all(test, windows))]
101mod program_binding_windows_tests;
102#[cfg(all(test, windows))]
103mod windows_channel_tests;
104pub use config_exit_failure::ModuleStopReason;
105#[cfg(all(test, unix))]
106mod config_exit_hold_tests;
107pub(crate) mod containment;
108#[cfg(all(test, unix))]
109mod containment_tests;
110mod slow_retry;
111use slow_retry::{FailedRetry, FailureContext};
112mod host_health;
113use host_health::HostHealth;
114
115#[cfg(all(test, unix))]
116mod mark_readiness_tests;
117#[cfg(all(test, unix))]
118mod slow_retry_tests;
119
120/// Command-line flag used by supervised modules to find subc.
121///
122/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
123/// The path points at the TCP+key connection file; it is not an ambient signal and
124/// is never inherited by standalone children.
125pub const SUBC_ARG: &str = "--subc";
126
127const DEFAULT_MAX_RESTARTS: u32 = 3;
128const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
129const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
130/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
131/// to contain a real crash loop (which respawns in seconds) and short enough
132/// that unrelated crashes hours apart never accumulate into a permanent stop.
133const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
134/// How long a drain waits for already-dispatched requests to finalize before
135/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
136/// not RPC-scale: the original 2s value silently cut nearly every real tool
137/// call at the fence, making the wait-for-finalize design decorative for the
138/// workloads it existed for. Quiescence short-circuits, so an idle module
139/// restarts immediately regardless of this value; the budget is spent only
140/// when a genuine in-flight request is worth finishing. Per-module override:
141/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
142/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
143/// where a stuck request will never settle).
144pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
145pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
146const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
147const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
148/// How long a drain waits after one busy-gauge probe completes before it asks
149/// again.
150///
151/// A drain re-probes on this short interval of its own rather than on the
152/// module's health cadence. The default cadence (30 s) equals the default
153/// drain ceiling, so a cadence-paced drain that found the module busy at its
154/// first probe never asked a second time: it waited out the whole ceiling and
155/// forced GOODBYE even after the gauges had read zero. The interval is counted
156/// from the moment the previous probe finished, and probes run one at a time,
157/// so a slow module is never asked more often than it can answer. Periodic
158/// health checks outside a drain keep using the configured cadence.
159const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
160/// How long a restart waits for the exited process's output readers.
161///
162/// The restart does not depend on either output reader finishing. A stderr
163/// reader still running at this bound is left running, and its later output goes
164/// into the exited process's own section of the stderr ring (see
165/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
166/// bound no longer decides whether a crash's last lines are kept: under load
167/// the reader may simply not have been scheduled yet, and cutting it there
168/// discarded exactly the lines that explained the crash.
169///
170/// What the bound still decides is when the tail starts reporting
171/// `Incomplete`: a pipe open past it usually means a descendant of the exited
172/// process still holds it, and the tail cannot claim to be whole until that
173/// pipe closes. That is also the only case in which the wait costs the restart
174/// anything, because a pipe with no other holder reaches EOF when the process
175/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
176/// `Captured` at EOF with nothing lost.
177///
178/// The stdout reader carries no ring, only the capture file. It also continues
179/// to EOF after this bound: child exit closes the writer but does not mean the
180/// reader has consumed all buffered bytes. Late output from either pipe can
181/// therefore reach the shared capture file after the successor starts.
182const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
183/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
184pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
185const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
186/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
187/// dropped because the subscriber stopped draining its frames. The detail's
188/// `first_undelivered_cursor` names the first event it did not receive; the
189/// client resubscribes from the last cursor it did receive.
190pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
191
192#[cfg(target_os = "macos")]
193struct PrivacyExec {
194    content_check: Option<tokio::task::JoinHandle<Result<(), crate::config_gate::GateError>>>,
195    gated: bool,
196    admission_binding: Option<ProgramBindingPublication>,
197    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
198    deadline: tokio::time::Instant,
199    expected: Option<subc_os::FileIdentity>,
200    trampoline: Option<subc_os::FileIdentity>,
201    signature: Option<Arc<subc_os::privacy_identity::SuspendedSignature>>,
202    module_id: String,
203    #[cfg(test)]
204    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
205    #[cfg(test)]
206    image_probe: Option<fn(u32) -> Option<subc_os::Observation>>,
207    #[cfg(test)]
208    image_read_delay: Option<Duration>,
209}
210
211#[cfg(all(test, target_os = "macos"))]
212pub(crate) fn test_privacy_trampoline() -> PathBuf {
213    // The fixture bin lives in the profile folder the unit-test executable was
214    // built into. This honors custom CARGO_TARGET_DIR too.
215    let path = cortexkit_test_support::cargo_profile_bin("privacy-trampoline-fixture");
216    // Without the fixture every macOS spawn is refused, and the tests that
217    // spawn fail later as a module in state Failed, which names the wrong
218    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
219    assert!(
220        path.exists(),
221        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
222         --bins --features test-support` or `cargo test -p subc-daemon` first",
223        path.display()
224    );
225    path
226}
227
228#[cfg(target_os = "macos")]
229fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
230    use std::io::Read;
231    let mut probe = subc_os::child_spawn::spawn(
232        std::process::Command::new(path)
233            .args(["__disclaim-exec", "--probe-suspended"])
234            .stdin(Stdio::null())
235            .stdout(Stdio::piped())
236            .stderr(Stdio::piped()),
237    )
238    .map_err(|error| {
239        format!(
240            "privacy trampoline probe failed for {}: {error}",
241            path.display()
242        )
243    })?;
244    let deadline = std::time::Instant::now() + Duration::from_secs(5);
245    let status = loop {
246        match probe.try_wait() {
247            Ok(Some(status)) => break status,
248            Ok(None) if std::time::Instant::now() < deadline => {
249                std::thread::sleep(Duration::from_millis(5))
250            }
251            result => {
252                let _ = probe.kill();
253                let _ = probe.wait();
254                return Err(format!(
255                    "privacy trampoline probe failed or timed out for {}: {result:?}",
256                    path.display()
257                ));
258            }
259        }
260    };
261    let mut answer = String::new();
262    if let Some(stdout) = probe.stdout.take() {
263        let _ = stdout.take(256).read_to_string(&mut answer);
264    }
265    if status.success() && answer.trim() == subc_os::privacy_identity::SUSPENDED_TRAMPOLINE_PROBE {
266        return Ok(());
267    }
268    let mut diagnostic = String::new();
269    if let Some(stderr) = probe.stderr.take() {
270        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
271    }
272    let cause = diagnostic
273        .trim()
274        .strip_prefix("ck-subc: own privacy identity refused: ")
275        .unwrap_or("binary does not implement the privacy trampoline protocol");
276    Err(format!(
277        "{cause}: probe of {} exited {status}",
278        path.display()
279    ))
280}
281
282#[cfg(target_os = "macos")]
283fn privacy_command(
284    spec: &ModuleSpec,
285    binding: &SpawnProgramBinding,
286    roster: &ChildRoster,
287    gated: bool,
288) -> Result<
289    (
290        Command,
291        Option<PrivacyExec>,
292        subc_os::privacy_identity::ExecAcknowledgement,
293    ),
294    SuperviseError,
295> {
296    let failure = |cause: String| {
297        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
298        SuperviseError::Spawn {
299            program: spec.program.clone(),
300            source: io::Error::other(cause),
301            cgroup_path: None,
302        }
303    };
304    let trampoline = roster.privacy_trampoline().map_err(failure)?;
305    let expected = Some(binding.identity);
306    let trampoline_image = subc_os::file_identity(&trampoline);
307    let signature = gated
308        .then(|| {
309            subc_os::privacy_identity::SuspendedSignature::capture(&binding._file, &binding.path)
310                .map(Arc::new)
311        })
312        .transpose()
313        .map_err(|error| failure(error.to_string()))?;
314    if expected == trampoline_image {
315        return Err(failure(
316            "privacy identity module executable is missing or is the trampoline itself".to_string(),
317        ));
318    }
319    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
320        .map_err(|error| failure(error.to_string()))?;
321    let reader =
322        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
323    let mut command = Command::new(&trampoline);
324    command
325        .arg(if gated {
326            "__disclaim-exec-suspended"
327        } else {
328            "__disclaim-exec"
329        })
330        .arg(ack.fd().to_string())
331        .arg(&binding.path);
332    Ok((
333        command,
334        Some(PrivacyExec {
335            gated,
336            admission_binding: None,
337            content_check: None,
338            reader,
339            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
340            expected,
341            trampoline: trampoline_image,
342            signature,
343            module_id: spec.module_id.clone(),
344            #[cfg(test)]
345            image_probe: None,
346            #[cfg(test)]
347            image_read_delay: spec
348                .env
349                .iter()
350                .find(|(key, _)| key == "SUBC_TEST_PRIVACY_IMAGE_DELAY_MS")
351                .map(|(_, millis)| Duration::from_millis(millis.parse().unwrap())),
352            #[cfg(test)]
353            confirmation_pause: spec
354                .env
355                .iter()
356                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
357                .map(|(_, address)| (address.clone(), None)),
358        }),
359        ack,
360    ))
361}
362
363#[cfg(unix)]
364type SpawnRegistrationChannel = std::os::unix::net::UnixStream;
365#[cfg(windows)]
366type SpawnRegistrationChannel = subc_os::registration_channel::RegistrationPipe;
367#[cfg(test)]
368type CapturedRegistrationChannels =
369    Arc<Mutex<Option<Vec<(Arc<RegistrationSpawn>, CapturedRegistrationEndpoint)>>>>;
370#[cfg(test)]
371struct CapturedRegistrationEndpoint {
372    _channel: SpawnRegistrationChannel,
373}
374#[cfg(test)]
375impl fmt::Debug for CapturedRegistrationEndpoint {
376    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
377        f.write_str("CapturedRegistrationEndpoint")
378    }
379}
380
381struct SupervisedChild {
382    registration_spawn: Option<Arc<RegistrationSpawn>>,
383    /// Owned daemon endpoint, retained for the per-spawn registration service.
384    #[cfg(any(unix, windows))]
385    registration_channel: Option<SpawnRegistrationChannel>,
386    live_program_binding: Option<Arc<content_binding::LiveProgramBinding>>,
387    registered_config_connection: Option<ConnectionId>,
388    admission_base: Option<subc_config::Digest>,
389    registration_declaration: Option<String>,
390    _program_binding: Option<ProgramBindingPublication>,
391    subtree_target: crate::subtree_resources::Target,
392    child: containment::ProcessChild,
393    exec: crate::activity::ExecOwner,
394    #[cfg(windows)]
395    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
396    #[cfg(target_os = "macos")]
397    privacy_exec: Option<PrivacyExec>,
398    /// Set once this launch's exec acknowledgement confirms the module image.
399    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
400    /// `subc_os::privacy_identity`), which then replaces itself with the
401    /// module. The supervisor owns and can kill that pid from spawn, but
402    /// status readers report it only after this latch is set, so nothing
403    /// reports the trampoline's image as the module's.
404    #[cfg(target_os = "macos")]
405    report_ready: Arc<OnceLock<()>>,
406    /// Supervisor failure detail retained for the terminal record, including
407    /// image refusals and first-HELLO timeouts after a successful spawn.
408    spawn_failure: Option<String>,
409    /// Set only after a kill requested because launch admission failed was
410    /// accepted, never for an operator, health, or shutdown kill.
411    admission_kill_sent: bool,
412    /// The protocol this process was launched with. A reload can store a new
413    /// launch spec with a different protocol, but that takes effect only at the
414    /// next spawn, so this process keeps being handled by the protocol it
415    /// actually speaks.
416    protocol: ModuleProtocol,
417    /// This container's cgroup name: a bounded module/slot label followed by a
418    /// unique suffix (when cgroup placement is on). Carried replacements reuse
419    /// the name; ordinary launches each get a fresh container. A
420    /// retired process in a slot may still be draining when a later one is
421    /// spawned into that slot, so the suffix keeps the later process out of
422    /// the retired one's cgroup, which is the domain a kill applies to.
423    #[cfg(target_os = "linux")]
424    module_id: String,
425    #[cfg(target_os = "linux")]
426    cgroup_placement: Option<subc_cgroup::Placement>,
427    /// The job for an ordinary, non-carried launch. For carried
428    /// launches the actor owns it and the child holds only a weak container link.
429    ///
430    /// Dropping this handle is what reaps a surviving tree when no supervisor
431    /// code runs — a daemon crash — because the job carries
432    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
433    ///
434    /// That limit is not crash-only, and the difference is worth knowing: a
435    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
436    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
437    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
438    /// module at once. Before this change they survived that, saw EOF on the
439    /// control socket, and ran their own teardown; Unix keeps that path
440    /// deliberately, so a module can seal a WAL or close a capture rather than
441    /// be killed mid-write. So this trades graceful teardown on every Windows
442    /// daemon stop for containment on a crash, which is the right way round
443    /// today: orphaned GPU workers are a reported, recurring problem, and the
444    /// modules that write most heavily do not run on Windows.
445    ///
446    /// The fix is a real Windows stop path — the daemon draining before it
447    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
448    /// reaches only what the drain left behind, which is what it should reach.
449    #[cfg(windows)]
450    job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
451    stdout_pump: Option<JoinHandle<()>>,
452    stderr_pump: Option<StderrPump>,
453    stderr_ring: Arc<Mutex<StderrRing>>,
454    /// Conservative monotonic bound sampled before `command.spawn()`, not when
455    /// identity recording finishes or the spawn generation is published.
456    spawned_at: Instant,
457    config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
458    /// Ordinary spawns must send their first HELLO within this budget. Once
459    /// accepted, a later disconnect must not restart the registration clock.
460    registration_deadline: Option<Instant>,
461    registration_watch: Option<RegistrationWatch>,
462    spawned_at_ms: u64,
463    spawned_from: PathBuf,
464    launch_spec_digest: LaunchSpecDigest,
465    spawned_file_identity: Option<SpawnedFileIdentity>,
466    process_start_time: Option<u64>,
467    process_identity: Option<ProcessIdentity>,
468    pid: u32,
469    /// This process's entry in the daemon's child roster, released when the
470    /// process is reaped or this handle is dropped.
471    roster_guard: Option<crate::child_roster::RosterGuard>,
472}
473
474impl SupervisedChild {
475    fn retire_registration(&self) {
476        // All exit observers enter here before cleanup can await. The retired
477        // program-binding flag and the registration's Dead state make repeated
478        // observations harmless, even when another task observes the group
479        // leader retained unreaped to reserve its process-group id.
480        if let Some(spawn) = &self.registration_spawn {
481            spawn.retire();
482        } else if let Some(binding) = &self._program_binding {
483            binding.retire();
484        }
485        #[cfg(unix)]
486        if let Some(channel) = &self.registration_channel {
487            let _ = channel.shutdown(std::net::Shutdown::Both);
488        }
489        #[cfg(windows)]
490        if let Some(channel) = &self.registration_channel {
491            if let Ok(shutdown) = channel.shutdown_handle() {
492                shutdown.shutdown();
493            }
494        }
495    }
496    fn registered_since_spawn(&self) -> bool {
497        self.registration_watch
498            .as_ref()
499            .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
500    }
501
502    fn awaiting_registration(&self) -> bool {
503        // Admission failure retires the channel before its kill becomes
504        // waitable. That closure must not replace the admission failure with
505        // a registration failure and spend crash restart budget.
506        let admission_failed = self.admission_kill_sent && self.spawn_failure.is_some();
507        !admission_failed && !self.registered_since_spawn()
508    }
509
510    fn id(&self) -> Option<u32> {
511        Some(self.pid)
512    }
513
514    fn process_identity(&self) -> Option<ProcessIdentity> {
515        self.process_identity
516    }
517
518    fn try_wait(&mut self) -> io::Result<Option<ExitStatus>> {
519        let result = self.child.try_wait();
520        if !matches!(result, Ok(None)) {
521            self.retire_registration();
522            #[cfg(any(unix, windows))]
523            self.registration_channel.take();
524            if let Some(binding) = &self._program_binding {
525                binding.retire();
526            }
527        }
528        result
529    }
530
531    async fn wait(&mut self) -> io::Result<ExitStatus> {
532        #[cfg(target_os = "macos")]
533        self.confirm_privacy_exec().await;
534        // The roster entry is NOT released here. A daemon shutdown waits for the
535        // roster to empty and then exits the process, so releasing at the reap
536        // let it exit before the exit handler wrote this child's terminal record
537        // (the stderr drain and snapshot update sit in between), and the
538        // shutdown's own `daemon_shutdown` record was intermittently lost. The
539        // caller releases it after recording the exit (`release_roster`), and
540        // dropping the handle releases it too.
541        let result = self.child.wait().await;
542        self.retire_registration();
543        #[cfg(any(unix, windows))]
544        self.registration_channel.take();
545        // All plain and swap waits pass here. Revoke the reaped child's nonce
546        // before cgroup cleanup, stderr draining or exit observation can await.
547        if let Some(binding) = &self._program_binding {
548            binding.retire();
549        }
550        #[cfg(all(test, target_os = "linux"))]
551        if let Some(binding) = &self._program_binding {
552            binding.pause_after_wait_for_test().await;
553        }
554        #[cfg(target_os = "linux")]
555        if result.is_ok() && self.child.container().is_none() {
556            if let Some(placement) = self.cgroup_placement.as_ref() {
557                // Ordinary children still own a per-spawn cgroup: returning
558                // from wait must leave it empty and removed, as before carry.
559                // Keep cleanup ownership across cancellation until it finishes.
560                cleanup_reaped_cgroup(placement, &self.module_id).await;
561                self.cgroup_placement = None;
562            }
563        }
564        result
565    }
566
567    async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
568        self.retire_registration();
569        // Idempotent fallback for cleanup paths that did not call wait.
570        if let Some(binding) = self._program_binding.take() {
571            binding.retire();
572        }
573        if self.child.container().is_some() {
574            if !carry {
575                self.child.terminate_container().await;
576            }
577        } else {
578            #[cfg(target_os = "linux")]
579            if let Some(placement) = self.cgroup_placement.as_ref() {
580                cleanup_reaped_cgroup(placement, &self.module_id).await;
581                self.cgroup_placement = None;
582            }
583        }
584        self.child.reap().await?;
585        self.release_roster();
586        Ok(())
587    }
588
589    #[cfg(target_os = "macos")]
590    async fn confirm_privacy_exec(&mut self) {
591        if self
592            .privacy_exec
593            .as_ref()
594            .is_some_and(|pending| pending.content_check.is_some())
595        {
596            self.finish_privacy_content_check().await;
597            return;
598        }
599        let Some(pending) = &mut self.privacy_exec else {
600            return;
601        };
602        let result = tokio::time::timeout_at(pending.deadline, async {
603            let mut record = Vec::new();
604            loop {
605                let mut ready = pending.reader.readable().await?;
606                let read = ready.try_io(|reader| {
607                    use std::io::Read;
608                    let mut reader = reader.get_ref();
609                    let mut buffer = [0u8; 256];
610                    reader.read(&mut buffer).map(|count| (count, buffer))
611                });
612                match read {
613                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
614                    Ok(Ok((count, buffer))) => {
615                        if record.len() + count > 1024 {
616                            return Err(io::Error::other(
617                                "privacy exec refusal record is too long",
618                            ));
619                        }
620                        record.extend_from_slice(&buffer[..count]);
621                    }
622                    Ok(Err(error)) => return Err(error),
623                    Err(_) => continue,
624                }
625            }
626        })
627        .await;
628        #[cfg(test)]
629        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
630            use tokio::io::{AsyncReadExt, AsyncWriteExt};
631            if stream.is_none() {
632                *stream = Some(
633                    tokio::net::TcpStream::connect(address.as_str())
634                        .await
635                        .unwrap(),
636                );
637                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
638                stream
639                    .as_mut()
640                    .unwrap()
641                    .write_all(&self.pid.to_ne_bytes())
642                    .await
643                    .unwrap();
644            }
645            let mut reply = [0];
646            stream
647                .as_mut()
648                .unwrap()
649                .read_exact(&mut reply)
650                .await
651                .unwrap();
652            pending.confirmation_pause = None;
653            if &reply == b"F" {
654                Ok(Err(io::Error::other("injected late exec refusal")))
655            } else {
656                assert_eq!(&reply, b"X");
657                result
658            }
659        } else {
660            result
661        };
662        // Keep the reader in self across await: select cancellation must not
663        // discard the handshake or reset its original five-second deadline.
664        // Use try_wait so a child reaped during confirmation has its program
665        // binding retired before another command can use its nonce.
666        let waited = if matches!(&result, Ok(Ok(record)) if record.is_empty()) {
667            Some(self.try_wait())
668        } else {
669            None
670        };
671        let pending = self.privacy_exec.as_ref().expect("pending exec");
672        let cause = match result {
673            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
674            Ok(Err(error)) => Some(format!(
675                "privacy identity exec acknowledgement failed: {error}"
676            )),
677            Ok(Ok(record)) if !record.is_empty() => Some(
678                std::str::from_utf8(&record)
679                    .ok()
680                    .and_then(|record| {
681                        record
682                            .trim()
683                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
684                    })
685                    .filter(|cause| !cause.is_empty())
686                    .unwrap_or("invalid privacy exec refusal record")
687                    .to_string(),
688            ),
689            Ok(Ok(_)) if !pending.gated => match waited.expect("exec EOF checked child") {
690                Ok(Some(_)) => None,
691                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
692                Ok(None) => {
693                    #[cfg(test)]
694                    if let Some(delay) = pending.image_read_delay {
695                        tokio::time::sleep(delay).await;
696                    }
697                    let image = observe_spawned_image(self.pid);
698                    #[cfg(test)]
699                    let image = pending.image_probe.map_or(image, |probe| probe(self.pid));
700                    // An unstamped module can re-exec before we wake to read its
701                    // image. Only the trampoline image proves exec never happened;
702                    // retain the observed image for later orphan identification.
703                    if let Some(image) = image.filter(|image| {
704                        image.executable.is_some() && image.executable != pending.trampoline
705                    }) {
706                        match self
707                            .roster_guard
708                            .as_ref()
709                            .map(|guard| guard.confirm_executable(image))
710                            .transpose()
711                        {
712                            Err(error) => Some(format!(
713                                "could not durably record confirmed module image: {error}"
714                            )),
715                            Ok(_) => {
716                                let _ = self.report_ready.set(());
717                                None
718                            }
719                        }
720                    } else if image.is_none()
721                        || image.is_some_and(|image| image.executable.is_none())
722                    {
723                        // Empty EOF acknowledged exec. An image disappearing
724                        // during observation is an ordinary module exit.
725                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
726                            Ok(result) => {
727                                self.retire_registration();
728                                self.registration_channel.take();
729                                if let Some(binding) = &self._program_binding {
730                                    binding.retire();
731                                }
732                                result.err().map(|error| format!("privacy identity trampoline wait failed: {error}"))
733                            }
734                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
735                        }
736                    } else {
737                        Some("privacy identity trampoline did not exec the module".to_string())
738                    }
739                }
740            },
741            Ok(Ok(_)) => match waited.expect("exec EOF checked child") {
742                Ok(Some(_)) => {
743                    Some("privacy identity target exited before suspended admission".to_string())
744                }
745                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
746                Ok(None) => {
747                    // CLOEXEC closes the ack just before the kernel reports the
748                    // suspended target. EOF alone must never admit a running image.
749                    let stopped = tokio::time::timeout_at(pending.deadline, async {
750                        loop {
751                            if subc_os::privacy_identity::SuspendedSignature::is_suspended(
752                                self.pid,
753                            )? {
754                                return Ok::<_, io::Error>(());
755                            }
756                            tokio::time::sleep(Duration::from_millis(1)).await;
757                        }
758                    })
759                    .await;
760                    match stopped {
761                        Err(_) => {
762                            Some("privacy identity target was not suspended within 5s".to_string())
763                        }
764                        Ok(Err(error)) => Some(format!(
765                            "privacy identity suspended target unavailable: {error}"
766                        )),
767                        Ok(Ok(())) => {
768                            let image = observe_spawned_image(self.pid);
769                            if image.is_some_and(|image| {
770                                image.executable.is_some()
771                                    && image.executable != pending.trampoline
772                                    && image.executable == pending.expected
773                            }) {
774                                None
775                            } else {
776                                Some("privacy identity trampoline executable mismatch: expected the suspended module program".to_string())
777                            }
778                        }
779                    }
780                }
781            },
782        };
783        #[cfg(test)]
784        if let Some(binding) = &self._program_binding {
785            binding.pause_after_wait_for_test().await;
786        }
787        if cause.is_none() && pending.gated {
788            let publication = pending.admission_binding.clone();
789            let live = self.live_program_binding.clone();
790            let signature = Arc::clone(pending.signature.as_ref().expect("gated signature"));
791            let pid = self.pid;
792            let gated = pending.gated;
793            self.privacy_exec
794                .as_mut()
795                .expect("pending exec")
796                .content_check = Some(tokio::task::spawn_blocking(move || {
797                signature.verify(pid).map_err(|error| {
798                    crate::config_gate::GateError::new("program_refused", error)
799                })?;
800                if !gated {
801                    return Ok(());
802                }
803                match (publication, live) {
804                    (Some(publication), _) => publication.verify(),
805                    (None, Some(live)) => live.check(),
806                    _ => Err(crate::config_gate::GateError::new(
807                        "config_delivery_deferred",
808                        "live program witness is unavailable",
809                    )),
810                }
811            }));
812            self.finish_privacy_content_check().await;
813        } else {
814            self.finish_privacy_exec(cause);
815        }
816    }
817
818    #[cfg(target_os = "macos")]
819    async fn finish_privacy_content_check(&mut self) {
820        let result = self
821            .privacy_exec
822            .as_mut()
823            .expect("pending exec")
824            .content_check
825            .as_mut()
826            .expect("pending content check")
827            .await;
828        let cause = match result {
829            Ok(Ok(())) => None,
830            Ok(Err(error)) => Some(format!("{}: {}", error.code, error.detail)),
831            Err(error) => Some(format!(
832                "config_delivery_deferred: content check stopped: {error}"
833            )),
834        };
835        self.finish_privacy_exec(cause);
836    }
837
838    #[cfg(target_os = "macos")]
839    fn finish_privacy_exec(&mut self, mut cause: Option<String>) {
840        let mut pending = self.privacy_exec.take().expect("pending exec");
841        if cause.is_none() && pending.gated {
842            let admission = (|| -> io::Result<()> {
843                let image = observe_spawned_image(self.pid)
844                    .filter(|image| image.executable == pending.expected)
845                    .ok_or_else(|| io::Error::other("suspended executable mismatch"))?;
846                if let Some(guard) = &self.roster_guard {
847                    guard.confirm_executable(image)?;
848                }
849                pending
850                    .signature
851                    .as_ref()
852                    .expect("gated signature")
853                    .resume(self.pid)?;
854                let _ = self.report_ready.set(());
855                info!(module_id = %pending.module_id, pid = self.pid,
856                    "verified suspended module resumed with own privacy identity");
857                Ok(())
858            })();
859            if let Err(error) = admission {
860                cause = Some(format!("program_refused: {error}"));
861            }
862        }
863        if cause.is_some() {
864            if let Some(spawn) = &self.registration_spawn {
865                spawn.refuse();
866            }
867            if let Some(binding) = &mut pending.admission_binding {
868                binding.reject();
869            }
870        } else if let Some(spawn) = &self.registration_spawn {
871            spawn.ready();
872        }
873        // Exec confirmation describes launch success, not roster retirement.
874        // Keep even a reaped child recorded until its terminal history exists.
875        self.exec.0.finish(if cause.is_some() {
876            crate::activity::ExecState::Refused
877        } else {
878            crate::activity::ExecState::CommittedExec
879        });
880        if let Some(cause) = cause {
881            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
882            self.spawn_failure = Some(
883                if pending.gated
884                    && (cause.contains("executable mismatch")
885                        || cause.starts_with("program_refused:"))
886                {
887                    format!("program_refused: {cause}")
888                } else if pending.gated {
889                    format!("config_delivery_deferred: {cause}")
890                } else {
891                    cause
892                },
893            );
894            // No resume signal is sent on failure; kill the entire fresh group
895            // even if a faulty trampoline did not honor the suspended protocol.
896            let _ = self.start_admission_kill();
897        }
898    }
899
900    /// Releases the leader's roster entry after recording and finalizing its
901    /// exit. A carried container retains its separate roster entry and, for a
902    /// Unix process group, its unreaped first leader that reserves the group ID.
903    fn release_roster(&mut self) {
904        self.roster_guard = None;
905    }
906
907    /// Kill the child and, where containment is available, its process tree.
908    ///
909    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
910    /// helper process leaked the helper — the Synapse embedding module's CUDA
911    /// worker holds the GPU allocation, so the leak cost VRAM until the next
912    /// restart of something else. Terminating the job reaches grandchildren that
913    /// a tree walk cannot, including one whose parent has already exited and
914    /// been reparented away.
915    ///
916    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
917    /// direct-child kill still decides the outcome, so containment can never
918    /// change whether a module is reported as stopped.
919    fn start_kill(&mut self) -> io::Result<()> {
920        self.retire_registration();
921        if let Some(container) = self.child.container() {
922            container.hard_kill();
923            return self.kill_leader();
924        }
925        #[cfg(windows)]
926        if let Some(job) = &self.job {
927            if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
928                debug!(
929                    error = %error,
930                    "job termination failed; the direct-child kill still owns the outcome"
931                );
932            }
933        }
934        #[cfg(target_os = "linux")]
935        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
936        self.child.start_kill()
937    }
938
939    fn kill_leader(&mut self) -> io::Result<()> {
940        self.retire_registration();
941        self.child.start_kill()
942    }
943
944    fn start_admission_kill(&mut self) -> io::Result<()> {
945        self.retire_registration();
946        #[cfg(windows)]
947        let result = {
948            // Preserve the leader's distinctive status before ending the whole
949            // container/job; descendants must not wait for exit finalization.
950            let result = self
951                .spawned_file_identity
952                .as_ref()
953                .and_then(|identity| identity.windows_image.as_ref())
954                .ok_or_else(|| io::Error::other("admission child handle unavailable"))
955                .and_then(|image| {
956                    image.process().terminate_with_exit_code(
957                        admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE as u32,
958                    )
959                });
960            if let Some(container) = self.child.container() {
961                container.hard_kill();
962            } else if let Some(job) = &self.job {
963                if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
964                    debug!(%error, "job termination failed after admission leader termination");
965                }
966            }
967            result
968        };
969        #[cfg(not(windows))]
970        let result = self.start_kill().map(|()| true);
971        #[cfg(unix)]
972        let result = {
973            // A leader can exit before its helpers. Still end the fresh group
974            // even when the direct-child kill reports that the leader is gone.
975            let group_sent = self.child.container().is_none()
976                && rustix::process::Pid::from_raw(self.pid as i32).is_some_and(|group| {
977                    rustix::process::kill_process_group(group, rustix::process::Signal::KILL)
978                        .is_ok()
979                });
980            if group_sent {
981                Ok(true)
982            } else {
983                result
984            }
985        };
986        self.admission_kill_sent |= result?;
987        Ok(())
988    }
989
990    async fn drain_stderr(&mut self, module_id: &str) {
991        if let Some(pump) = self.stdout_pump.take() {
992            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
993        }
994
995        let Some(pump) = self.stderr_pump.take() else {
996            return;
997        };
998        settle_stderr_pump(
999            module_id,
1000            &self.stderr_ring,
1001            pump,
1002            STDERR_PUMP_DRAIN_TIMEOUT,
1003        )
1004        .await;
1005    }
1006}
1007
1008/// Bound the restart's wait, not the lifetime of the exited child's stdout
1009/// reader. Slow pipe reads or capture writes must not turn the bound into lost
1010/// output; dropping the handle lets the reader finish at EOF, like stderr.
1011async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
1012    match timeout(bound, &mut pump).await {
1013        Ok(Ok(())) => {}
1014        Ok(Err(error)) => {
1015            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
1016        }
1017        Err(_) => {
1018            drop(pump);
1019            warn!(
1020                module_id,
1021                waited = ?bound,
1022                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
1023            );
1024        }
1025    }
1026}
1027
1028/// The reader task for one process's stderr, with the ring generation its
1029/// lines are attributed to.
1030struct StderrPump {
1031    task: JoinHandle<()>,
1032    generation: u64,
1033}
1034
1035/// Retire an exited process's stderr reader and wait up to `bound` for it to
1036/// reach EOF. A reader still running at the bound is detached, not stopped: it
1037/// keeps filling the exited process's section of the ring until its pipe
1038/// closes, and the tail reads `Incomplete` until then. See
1039/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
1040async fn settle_stderr_pump(
1041    module_id: &str,
1042    ring: &Arc<Mutex<StderrRing>>,
1043    pump: StderrPump,
1044    bound: Duration,
1045) {
1046    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
1047    let StderrPump {
1048        mut task,
1049        generation,
1050    } = pump;
1051    lock().retire_pump(generation);
1052    match timeout(bound, &mut task).await {
1053        Ok(Ok(())) => {}
1054        Ok(Err(err)) => {
1055            let mut ring = lock();
1056            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
1057            ring.finish_pump(generation);
1058            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
1059        }
1060        Err(_) => {
1061            // Dropping the handle detaches the task; it ends at EOF on its pipe.
1062            drop(task);
1063            lock().mark_pump_late(
1064                generation,
1065                format!(
1066                    "stderr of the exited process had not reached EOF {bound:?} after it was \
1067                     retired (a descendant may still hold the pipe open); lines it still \
1068                     writes are kept in that process's section"
1069                ),
1070            );
1071            warn!(
1072                module_id,
1073                waited = ?bound,
1074                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
1075            );
1076        }
1077    }
1078}
1079
1080fn registration_release_events() -> &'static watch::Sender<u64> {
1081    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
1082    EVENTS.get_or_init(|| {
1083        let (sender, _receiver) = watch::channel(0);
1084        sender
1085    })
1086}
1087
1088pub(crate) fn notify_registration_release() {
1089    let events = registration_release_events();
1090    let next_generation = (*events.borrow()).wrapping_add(1);
1091    events.send_replace(next_generation);
1092}
1093
1094/// How to launch one singleton module process.
1095#[derive(Debug, Clone, PartialEq, Eq)]
1096pub struct ModuleSpec {
1097    pub module_id: String,
1098    pub program: PathBuf,
1099    /// A bundled Bun ESM file. The program is always the installed runner.
1100    pub script: Option<PathBuf>,
1101    pub args: Vec<String>,
1102    pub env: Vec<(String, String)>,
1103    /// When true this is a reserved module: each spawn gets a fresh one-time launch
1104    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
1105    /// process can register this module_id (a security-boundary module like the
1106    /// credential vault must not be impersonable while it is down/restarting).
1107    pub reserved: bool,
1108    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
1109    /// Prefixes come from daemon config and must end in `:` before they reach the
1110    /// supervisor; the owner module's current spawn nonce authorizes claims under
1111    /// each prefix.
1112    pub reserved_prefixes: Vec<String>,
1113    /// The wire protocol this module speaks, as DECLARED in daemon config.
1114    ///
1115    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
1116    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
1117    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
1118    /// and NO launch nonce, and a clean exit the daemon did not request is
1119    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
1120    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
1121    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
1122    /// because a process ignores an environment variable it does not read.
1123    ///
1124    /// The argument is the part that cannot be "harmless to a process that
1125    /// ignores it": a stock binary exits on an unknown flag before it listens
1126    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
1127    /// first conformance run against this mode found it. The nonce is withheld
1128    /// because a process that will never present it gains nothing from holding
1129    /// it, and a secret in the environment of a process that does not need it is
1130    /// a leak surface for no benefit.
1131    pub protocol: ModuleProtocol,
1132    /// Whether two processes of this module may run at once, which is what a
1133    /// blue/green swap does for the length of its overlap. Declared in daemon
1134    /// config because the daemon must be able to answer it while the module is
1135    /// down, and so a module cannot talk itself into it after registering.
1136    pub overlap: ModuleOverlap,
1137    /// Keep module-started helpers in the same container across replacements.
1138    /// This is a daemon-config policy, fixed for the lifetime of a container;
1139    /// stopping the module or losing its supervisor still destroys the container.
1140    pub carry_containment: bool,
1141}
1142
1143/// Only the configured launch inputs enter this value; spawned connection paths,
1144/// nonces and inherited environment never do. Its bytes remain daemon-private.
1145#[derive(Clone, PartialEq, Eq)]
1146pub(crate) struct LaunchSpecDigest([u8; 32]);
1147
1148impl fmt::Debug for LaunchSpecDigest {
1149    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1150        f.write_str("LaunchSpecDigest([redacted])")
1151    }
1152}
1153
1154impl LaunchSpecDigest {
1155    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
1156        fn field(hash: &mut Sha256, bytes: &[u8]) {
1157            hash.update((bytes.len() as u64).to_le_bytes());
1158            hash.update(bytes);
1159        }
1160        let mut hash = Sha256::new();
1161        if let Some(script) = &spec.script {
1162            field(&mut hash, b"bun-script");
1163            field(&mut hash, script.as_os_str().as_encoded_bytes());
1164        }
1165        if let Some(program) = spec.program.to_str() {
1166            field(&mut hash, b"utf8-program");
1167            field(&mut hash, program.as_bytes());
1168        } else {
1169            // The daemon config is UTF-8, but in-process callers may provide
1170            // native paths: lossy conversion would collapse distinct programs.
1171            #[cfg(unix)]
1172            {
1173                use std::os::unix::ffi::OsStrExt;
1174                field(&mut hash, b"unix-program");
1175                field(&mut hash, spec.program.as_os_str().as_bytes());
1176            }
1177            #[cfg(windows)]
1178            {
1179                use std::os::windows::ffi::OsStrExt;
1180                field(&mut hash, b"windows-program");
1181                let bytes = spec
1182                    .program
1183                    .as_os_str()
1184                    .encode_wide()
1185                    .flat_map(u16::to_le_bytes)
1186                    .collect::<Vec<_>>();
1187                field(&mut hash, &bytes);
1188            }
1189            #[cfg(not(any(unix, windows)))]
1190            field(&mut hash, spec.program.to_string_lossy().as_bytes());
1191        }
1192        hash.update((spec.args.len() as u64).to_le_bytes());
1193        for arg in &spec.args {
1194            field(&mut hash, arg.as_bytes());
1195        }
1196        let mut env = spec.env.iter().collect::<Vec<_>>();
1197        env.sort_unstable();
1198        hash.update((env.len() as u64).to_le_bytes());
1199        for (key, value) in env {
1200            field(&mut hash, key.as_bytes());
1201            field(&mut hash, value.as_bytes());
1202        }
1203        field(
1204            &mut hash,
1205            match spec.protocol {
1206                ModuleProtocol::Subc => b"subc",
1207                ModuleProtocol::None => b"none",
1208            },
1209        );
1210        Self(hash.finalize().into())
1211    }
1212}
1213
1214/// Whether a module tolerates a second process of itself running alongside.
1215///
1216/// Most modules are single-writer on their store (a WAL, a capture log, a
1217/// resident index behind a writer barrier), and two processes on one store
1218/// corrupt it. So a swap, which overlaps the old and new process by design,
1219/// is refused unless the module's config opts in with `overlap: "safe"`.
1220#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1221pub enum ModuleOverlap {
1222    /// Never run two processes of this module at once. The default.
1223    #[default]
1224    Exclusive,
1225    /// The module has said a second process of itself is harmless for the
1226    /// length of a swap.
1227    ///
1228    /// Declare it only if a second instance can run for a few seconds without
1229    /// touching ANY single-writer store: every database, WAL, index, projector
1230    /// and scheduled job the module owns. A lease on part of that state is not
1231    /// enough. broca's session lease guards WAL appends while its run index, its
1232    /// store projector and its archive fold timer (which unlinks live WAL files)
1233    /// stay single-writer, so broca is exclusive despite holding a lease. The
1234    /// refusal only fires after this has been decided, so the decision is the
1235    /// check.
1236    Safe,
1237}
1238
1239impl ModuleOverlap {
1240    pub fn as_str(self) -> &'static str {
1241        match self {
1242            Self::Exclusive => "exclusive",
1243            Self::Safe => "safe",
1244        }
1245    }
1246}
1247
1248/// Environment variable telling a spawned module which case it was started
1249/// for, before it sends HELLO. Only a swap candidate carries it, as
1250/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
1251///
1252/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
1253/// longer because nobody waits on it, while a plain restart must flip ready
1254/// quickly because callers see `module_warming` until it does. Absence means
1255/// plain restart, the safe reading. The daemon trusts nothing about it; the
1256/// candidate is proven by its launch nonce at HELLO.
1257pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
1258/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
1259pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
1260/// How long a swap waits for its candidate to register and declare itself
1261/// ready when the operator does not say. A module warming as a swap candidate
1262/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
1263/// daemon allows that plus time to start the process and send HELLO.
1264pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
1265
1266/// Bounded restart policy for crash exits.
1267///
1268/// `max_restarts` is the number of replacement processes allowed after the
1269/// initial spawn WITHIN `window`. After that many crash restarts inside one
1270/// window the module enters [`ModuleState::Failed`] and the supervisor stops
1271/// the crash loop.
1272///
1273/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
1274/// and that only survived because crashes were rare: a module that crashed
1275/// three times across a week was disabled forever by crashes that had nothing
1276/// to do with each other. That stopped being survivable once modules began
1277/// exiting non-zero whenever the daemon's connection to them drops, because
1278/// then every daemon-side connection drop spends a unit of the same budget and
1279/// one flappy hour permanently stops a healthy module. Restarts older than
1280/// `window` release their slot, so a module that crashed twice yesterday has a
1281/// full budget today, while a genuine crash loop -- which is fast by
1282/// definition -- still reaches the cap and stops.
1283#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1284pub struct RestartPolicy {
1285    pub max_restarts: u32,
1286    /// Base delay before a crash replacement. The actual delay escalates with
1287    /// the number of recent crash replacements and is capped by `max_backoff`.
1288    pub backoff: Duration,
1289    /// Maximum delay before a crash replacement.
1290    pub max_backoff: Duration,
1291    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
1292    /// budget effectively infinite (nothing is ever in-window), which is why
1293    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
1294    pub window: Duration,
1295}
1296
1297impl RestartPolicy {
1298    /// A policy with the default crash window. Callers that care about the
1299    /// window say so with [`Self::with_window`]; the ones that do not are
1300    /// asking for the standard rate limit, not for no limit.
1301    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
1302        Self {
1303            max_restarts,
1304            backoff,
1305            max_backoff: DEFAULT_MAX_BACKOFF,
1306            window: DEFAULT_RESTART_WINDOW,
1307        }
1308    }
1309
1310    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
1311        self.max_backoff = max_backoff;
1312        self
1313    }
1314
1315    pub fn with_window(mut self, window: Duration) -> Self {
1316        self.window = window;
1317        self
1318    }
1319
1320    /// Calculate the capped exponential delay for the next crash replacement.
1321    /// `restart_in_window` is zero for the first replacement after an operator
1322    /// action (restart, reload, re-enable) cleared the crash ring, or after all
1323    /// older crash replacements have aged out of the window.
1324    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
1325        if self.backoff.is_zero() || self.max_backoff.is_zero() {
1326            return Duration::ZERO;
1327        }
1328
1329        let mut delay = self.backoff;
1330        for _ in 0..restart_in_window {
1331            if delay >= self.max_backoff {
1332                return self.max_backoff;
1333            }
1334            delay = delay
1335                .checked_mul(10)
1336                .unwrap_or(self.max_backoff)
1337                .min(self.max_backoff);
1338        }
1339        delay.min(self.max_backoff)
1340    }
1341
1342    /// The one sentence that explains a budget-exhausted stop, used for both the
1343    /// log line and the terminal record so the two cannot drift. It names the
1344    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
1345    /// exactly what this budget is not.
1346    fn budget_exhausted_detail(&self) -> String {
1347        format!(
1348            "crash budget exhausted: max_restarts={} within window_secs={}",
1349            self.max_restarts,
1350            self.window.as_secs()
1351        )
1352    }
1353}
1354
1355impl Default for RestartPolicy {
1356    fn default() -> Self {
1357        Self {
1358            max_restarts: DEFAULT_MAX_RESTARTS,
1359            backoff: DEFAULT_BACKOFF,
1360            max_backoff: DEFAULT_MAX_BACKOFF,
1361            window: DEFAULT_RESTART_WINDOW,
1362        }
1363    }
1364}
1365
1366#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1367struct CrashRestartSchedule {
1368    restart_in_window: u32,
1369    delay: Duration,
1370}
1371
1372/// Whether the daemon itself will bring this module back after the exit being
1373/// handled: it is enabled AND its in-window crash restarts are below the cap.
1374///
1375/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
1376/// the window are dropped here rather than by a timer, so the count is right
1377/// the moment somebody asks and no bookkeeping runs for idle modules.
1378fn daemon_will_restart(
1379    state: &mut SupervisorSnapshot,
1380    policy: &RestartPolicy,
1381    now: Instant,
1382) -> bool {
1383    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1384}
1385
1386const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1387const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1388const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1389const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1390
1391#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1392pub enum HealthAction {
1393    Report,
1394    Restart,
1395    Alert,
1396}
1397
1398impl fmt::Display for HealthAction {
1399    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1400        f.write_str(match self {
1401            Self::Report => "report",
1402            Self::Restart => "restart",
1403            Self::Alert => "alert",
1404        })
1405    }
1406}
1407
1408#[derive(Debug, Clone, PartialEq, Eq)]
1409pub struct HealthConfig {
1410    /// Optional loopback HTTP endpoint for a managed non-wire process.
1411    /// Changing it applies live on rescan; the process protocol changes only
1412    /// at its next spawn.
1413    pub http: Option<String>,
1414    pub cadence: Duration,
1415    pub deadline: Duration,
1416    pub failure_threshold: u32,
1417    pub on_degraded: HealthAction,
1418    pub on_failing: HealthAction,
1419    pub critical: bool,
1420}
1421
1422impl Default for HealthConfig {
1423    fn default() -> Self {
1424        Self {
1425            http: None,
1426            cadence: DEFAULT_HEALTH_CADENCE,
1427            deadline: DEFAULT_HEALTH_DEADLINE,
1428            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1429            on_degraded: HealthAction::Report,
1430            on_failing: HealthAction::Report,
1431            critical: false,
1432        }
1433    }
1434}
1435
1436/// The supervisor's view of one module's health, relayed to clients over
1437/// channel-0 and rendered by `ck health`.
1438///
1439/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1440/// stated here rather than only at the wire type a consumer reads. A reader can
1441/// look up what `None` means; only a writer can silently change it, and the
1442/// writer has no reason to go looking at a downstream contract before editing.
1443///
1444/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1445/// back to `None` on re-registration precisely so a respawned module does not
1446/// carry its predecessor's timestamp — so an old value and an absent one call for
1447/// opposite readings, and anything that defaulted this to a number would make a
1448/// never-probed module indistinguishable from one probed at the epoch.
1449///
1450/// `detail` and `metrics` are `None` when the module published none on this
1451/// probe, which does not mean it reported nothing wrong — it is also the shape
1452/// when the probe never reached it. `last_probe_ms` is what separates those.
1453#[derive(Debug, Clone, PartialEq)]
1454pub struct ModuleHealthStatus {
1455    pub status: SupervisorHealthStatus,
1456    pub last_probe_ms: Option<u64>,
1457    pub detail: Option<String>,
1458    pub metrics: Option<Value>,
1459    pub consecutive_failures: u32,
1460    /// Number of replies received after a recurring health probe's deadline.
1461    /// Unlike a timeout, every increment proves the module was alive.
1462    pub late_answer_count: u64,
1463    /// End-to-end latency of the newest late reply, measured from probe start.
1464    pub last_late_answer_latency_ms: Option<u64>,
1465    pub last_action: Option<String>,
1466    /// Set together with `last_action`; the pair moves as one, and both being
1467    /// absent means no escalation has ever been taken rather than that the last
1468    /// one succeeded.
1469    pub last_action_ms: Option<u64>,
1470}
1471
1472impl Default for ModuleHealthStatus {
1473    fn default() -> Self {
1474        Self {
1475            status: SupervisorHealthStatus::Unknown,
1476            last_probe_ms: None,
1477            detail: None,
1478            metrics: None,
1479            consecutive_failures: 0,
1480            late_answer_count: 0,
1481            last_late_answer_latency_ms: None,
1482            last_action: None,
1483            last_action_ms: None,
1484        }
1485    }
1486}
1487
1488/// Typed lifecycle state for a supervised module.
1489#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1490pub enum ModuleState {
1491    Starting,
1492    Running,
1493    Unresponsive,
1494    Restarting,
1495    Draining,
1496    Stopped,
1497    Failed,
1498    Disabled,
1499}
1500
1501impl fmt::Display for ModuleState {
1502    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1503        f.write_str(match self {
1504            Self::Starting => "starting",
1505            Self::Running => "running",
1506            Self::Unresponsive => "unresponsive",
1507            Self::Restarting => "restarting",
1508            Self::Draining => "draining",
1509            Self::Stopped => "stopped",
1510            Self::Failed => "failed",
1511            Self::Disabled => "disabled",
1512        })
1513    }
1514}
1515
1516/// Supervisor classification of a child-process exit.
1517#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1518pub enum ExitKind {
1519    Clean,
1520    Crash,
1521    DeliberateSeverance,
1522}
1523
1524impl From<ExitKind> for TerminalExitKind {
1525    fn from(kind: ExitKind) -> Self {
1526        match kind {
1527            ExitKind::Clean => Self::Clean,
1528            ExitKind::Crash => Self::Crash,
1529            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1530        }
1531    }
1532}
1533
1534/// Exact process identity retained when a supervised module registers its
1535/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1536#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1537pub(crate) struct ProcessIdentity {
1538    pub(crate) pid: u32,
1539    pub(crate) start_time: u64,
1540}
1541
1542/// Last observed child exit, if any.
1543#[derive(Debug, Clone, PartialEq, Eq)]
1544pub struct ExitReport {
1545    pub kind: ExitKind,
1546    pub code: Option<i32>,
1547    pub signal: Option<i32>,
1548    pub at_ms: u64,
1549}
1550
1551/// Point-in-time module status answerable by subc without forwarding to the
1552/// module process.
1553#[derive(Debug, Clone, PartialEq)]
1554pub struct ModuleStatus {
1555    pub(crate) admission_refused: Option<String>,
1556    pub(crate) legacy_registration_spawn: bool,
1557    pub(crate) registration_declaration: Option<String>,
1558    pub module_id: String,
1559    /// Retained configuration-exit infrastructure failure. Cleared only after a
1560    /// successful launch; an ordinary stop has no infrastructure stop reason.
1561    pub stop_reason: Option<ModuleStopReason>,
1562    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1563    pub waiting_for_previous_pid: Option<u32>,
1564    /// Why the previous instance still prevents a new start.
1565    pub waiting_for_previous_detail: Option<String>,
1566    /// The daemon's Windows per-spawn record: `pipe` after the process it
1567    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1568    /// List/status reports use this record, not the source the module sends
1569    /// in its registration message's manifest.provenance.launch_nonce_source.
1570    pub launch_nonce_source: Option<String>,
1571    pub state: ModuleState,
1572    pub enabled: bool,
1573    pub process_alive: bool,
1574    pub registration_active: bool,
1575    /// The module's declared wire protocol, carried beside `live` because it is
1576    /// what makes `live` readable: the two fields answer one question together.
1577    /// While a process is alive this is its launch declaration, not a later
1578    /// pending-reload edit. When down it is the configured next launch protocol.
1579    pub protocol: ModuleProtocol,
1580    /// Whether the module is serving, under the strongest definition the daemon
1581    /// can assert for its protocol.
1582    ///
1583    /// A subc module must also be REGISTERED: its process being alive says
1584    /// nothing about whether it can take a request. A `protocol: "none"` module
1585    /// never registers, so that term is dropped and this falls back to "enabled,
1586    /// running, and the process the daemon launched is alive" -- which is all
1587    /// the daemon observes about a process that speaks no subc wire. It stays a
1588    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1589    /// rather than printing it bare.
1590    pub live: bool,
1591    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1592    /// restarts have already released their slot, so this count can go down
1593    /// without anybody touching the module.
1594    pub restart_count: u32,
1595    /// Replacement processes spawned over this module's entire supervisor lifetime;
1596    /// unlike `restart_count`, this value is never reset by an operator action
1597    /// and never falls out of a window.
1598    pub lifetime_restarts: u32,
1599    pub spawn_generation: u64,
1600    /// Retained daemon-local outcome; the legacy control response omits it.
1601    pub swap_outcome: Option<SwapOutcome>,
1602    /// The budget `restart_count` is spent against. Carried alongside the count
1603    /// because the count alone does not say how close the module is to being
1604    /// disabled, and reporting one without the other is what makes an
1605    /// about-to-be-retired module look ordinary.
1606    pub max_restarts: u32,
1607    /// The span `restart_count` is counted over. Carried with the pair above for
1608    /// the same reason they are carried together: "2 of 3" means one thing for a
1609    /// ten-minute window and something else entirely for a lifetime.
1610    pub restart_window: Duration,
1611    /// Effective drain and restart timing policy used by this running module.
1612    /// These values are carried together with the restart budget so status
1613    /// readers can compare configured intent with what the supervisor applied.
1614    pub drain_timeout: Duration,
1615    pub restart_backoff: Duration,
1616    pub restart_max_backoff: Duration,
1617    /// The module's process. On macOS this stays absent while the `ck-subc`
1618    /// launch trampoline is still running in that pid, and appears once the
1619    /// exec acknowledgement confirms the module image has replaced it. Launch
1620    /// time and the supervisor's own hold on the process are unaffected.
1621    pub pid: Option<u32>,
1622    pub spawned_at_ms: Option<u64>,
1623    pub spawned_from: Option<PathBuf>,
1624    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1625    pub process_start_time: Option<u64>,
1626    pub last_exit: Option<ExitReport>,
1627    pub health: ModuleHealthStatus,
1628}
1629
1630#[cfg(test)]
1631#[derive(Debug, Clone, Copy, PartialEq)]
1632struct ActorSelectCheckpoint {
1633    generation: u64,
1634    turn: u64,
1635    registered_connection: Option<ConnectionId>,
1636    next_probe_at: Option<Instant>,
1637    wake_after: Option<Duration>,
1638}
1639
1640#[derive(Debug, Clone, PartialEq)]
1641struct SupervisorSnapshot {
1642    admission_refused: Option<String>,
1643    /// Distinguishes an admission-parked actor from an unrelated terminal exit.
1644    config_hold_parked: bool,
1645    /// Preserve daemon attribution for a paused start so a failing release can
1646    /// schedule the next slow retry instead of treating the start as operator work.
1647    config_hold_daemon_initiated: bool,
1648    /// The approval that owns the parked attempt, not whichever approval is latest.
1649    config_hold_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1650    /// Retain an approved launch failure across competing reconciliation passes.
1651    approved_start_failure: Option<(crate::config_gate::runtime::ApprovalIdentity, String)>,
1652    legacy_registration_spawn: bool,
1653    registration_declaration: Option<String>,
1654    stop_reason: Option<ModuleStopReason>,
1655    config_exit_failed_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1656    subtree_target: crate::subtree_resources::Target,
1657    failed_retry: FailedRetry,
1658    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1659    #[cfg(windows)]
1660    pipe_consumed: Option<Arc<OnceLock<()>>>,
1661    /// Counts running-child loop turns, not executor polls of a parked wait.
1662    #[cfg(test)]
1663    actor_turns: u64,
1664    /// Running select after exec confirmation and with no unconsumed registry
1665    /// event. Recording a spawn or starting a loop turn is not this barrier.
1666    #[cfg(test)]
1667    actor_select: Option<ActorSelectCheckpoint>,
1668    state: ModuleState,
1669    enabled: bool,
1670    process_alive: bool,
1671    spawned_protocol: Option<ModuleProtocol>,
1672    spawn_failure: Option<String>,
1673    waiting_for_previous_pid: Option<u32>,
1674    waiting_for_previous_detail: Option<String>,
1675    /// When each crash restart was spent, oldest first. This IS the crash
1676    /// budget: its in-window length is the count an operator sees and the count
1677    /// the restart decision is made against, so there is no second counter that
1678    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1679    /// operator actions that used to zero the old lifetime counter.
1680    crash_restarts: VecDeque<Instant>,
1681    lifetime_restarts: u32,
1682    /// Successful child spawns in this daemon incarnation.
1683    ///
1684    /// `lifetime_restarts` was considered and rejected: it starts at zero
1685    /// (line 640), successful initial/operator spawns in `set_running` do not
1686    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1687    /// increments before a successful replacement exists (lines 604, 3846,
1688    /// and 3921), so a failed spawn can consume it. This counter moves only
1689    /// when a live PID is accepted below.
1690    spawn_generation: u64,
1691    swap_outcome: Option<SwapOutcome>,
1692    pid: Option<u32>,
1693    #[cfg(target_os = "macos")]
1694    report_ready: Option<Arc<OnceLock<()>>>,
1695    /// Last reaped child, retained after current process facts are cleared.
1696    reaped_pid: Option<u32>,
1697    /// Whether a replacement has already been scheduled for the current process.
1698    respawn_pending: bool,
1699    /// A second restart is waiting for the replacement already scheduled.
1700    coalesced_restart_pending: bool,
1701    spawned_at_ms: Option<u64>,
1702    spawned_from: Option<PathBuf>,
1703    launch_spec_digest: Option<LaunchSpecDigest>,
1704    spawned_file_identity: Option<SpawnedFileIdentity>,
1705    process_start_time: Option<u64>,
1706    deliberate_severance: Option<ProcessIdentity>,
1707    last_exit: Option<ExitReport>,
1708    /// Diagnostic attached to the next drain's terminal record, if any.
1709    drain_disposition_detail: Option<String>,
1710    health: ModuleHealthStatus,
1711    last_probe_evidence: Option<HealthProbeEvidence>,
1712    /// Whether the current process was started as a swap candidate and so
1713    /// lives in the module's alternate cgroup. The next swap's candidate takes
1714    /// the other one, so the two processes of a swap never share a cgroup. A
1715    /// plain spawn always uses the primary cgroup.
1716    in_alternate_slot: bool,
1717    /// Whether the current `Draining` state ends in a replacement process
1718    /// (restart, reload, health restart) rather than a stop. Only meaningful
1719    /// while `state` is `Draining`; every entry into that state rewrites it.
1720    /// It is what lets route.open answer the retryable `module_reloading` to a
1721    /// consumer that reaches a still-registered process mid-restart, instead of
1722    /// the `supervisor_not_live` a stop or disable deserves.
1723    draining_to_replace: bool,
1724    /// Whether a configuration update has been applied since the current
1725    /// process was spawned, so that process runs an older spec than the one
1726    /// the supervisor now holds. A queued restart is only coalesced into a
1727    /// fresher process when this is false: a restart requested to pick up a
1728    /// new configuration must not be satisfied by a process that predates it.
1729    configuration_updated_since_spawn: bool,
1730}
1731
1732impl SupervisorSnapshot {
1733    /// The pid that status, provenance and resource readings may report. While
1734    /// the launch trampoline still runs in the pid, reading its executable or
1735    /// resource use would describe `ck-subc`, not the module, so none is
1736    /// reported until the exec acknowledgement confirms the module image.
1737    fn reported_pid(&self) -> Option<u32> {
1738        #[cfg(target_os = "macos")]
1739        if self
1740            .report_ready
1741            .as_ref()
1742            .is_some_and(|ready| ready.get().is_none())
1743        {
1744            return None;
1745        }
1746        self.pid
1747    }
1748
1749    fn starting() -> Self {
1750        Self::new(ModuleState::Starting, true)
1751    }
1752
1753    fn disabled() -> Self {
1754        Self::new(ModuleState::Disabled, false)
1755    }
1756
1757    /// Crash restarts still inside `window`, having dropped the ones that are
1758    /// not. Pruning on read is what makes the budget a rate: an instant older
1759    /// than the window stops holding a slot the moment anybody counts.
1760    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1761        while let Some(oldest) = self.crash_restarts.front() {
1762            if now.duration_since(*oldest) > window {
1763                self.crash_restarts.pop_front();
1764            } else {
1765                break;
1766            }
1767        }
1768        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1769    }
1770
1771    /// Spend one unit of the crash budget and record the restart in the ledger.
1772    ///
1773    /// The ring is bounded by the cap because more than `max_restarts` in-window
1774    /// instants can never be reached (the caller refuses the restart first), so
1775    /// anything beyond that is an unbounded queue waiting to happen.
1776    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1777        self.crash_restarts.push_back(now);
1778        while self.crash_restarts.len() > policy.max_restarts as usize {
1779            self.crash_restarts.pop_front();
1780        }
1781        self.lifetime_restarts += 1;
1782    }
1783
1784    /// Reserve one crash-restart slot and calculate the delay before respawning.
1785    /// The count is captured before recording this restart, so the first retry
1786    /// uses the base delay and each later in-window retry escalates once.
1787    fn next_crash_restart(
1788        &mut self,
1789        policy: &RestartPolicy,
1790        now: Instant,
1791    ) -> Option<CrashRestartSchedule> {
1792        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1793        if restart_in_window >= policy.max_restarts {
1794            return None;
1795        }
1796        self.record_crash_restart(policy, now);
1797        Some(CrashRestartSchedule {
1798            restart_in_window,
1799            delay: policy.delay_for_restart(restart_in_window),
1800        })
1801    }
1802
1803    /// Give the module its full budget back, as an operator restart, reload, or
1804    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1805    /// ledger of what actually happened, and an operator action does not unmake
1806    /// the crashes.
1807    fn clear_crash_restarts(&mut self) {
1808        self.crash_restarts.clear();
1809    }
1810
1811    fn new(state: ModuleState, enabled: bool) -> Self {
1812        Self {
1813            admission_refused: None,
1814            config_hold_parked: false,
1815            config_hold_daemon_initiated: false,
1816            config_hold_approval: None,
1817            approved_start_failure: None,
1818            legacy_registration_spawn: false,
1819            registration_declaration: None,
1820            stop_reason: None,
1821            config_exit_failed_approval: None,
1822            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1823            failed_retry: FailedRetry::default(),
1824            exec_outcome: None,
1825            #[cfg(test)]
1826            actor_turns: 0,
1827            #[cfg(test)]
1828            actor_select: None,
1829            state,
1830            enabled,
1831            process_alive: false,
1832            #[cfg(windows)]
1833            pipe_consumed: None,
1834            spawned_protocol: None,
1835            spawn_failure: None,
1836            waiting_for_previous_pid: None,
1837            waiting_for_previous_detail: None,
1838            crash_restarts: VecDeque::new(),
1839            lifetime_restarts: 0,
1840            spawn_generation: 0,
1841            swap_outcome: None,
1842            pid: None,
1843            #[cfg(target_os = "macos")]
1844            report_ready: None,
1845            reaped_pid: None,
1846            respawn_pending: false,
1847            coalesced_restart_pending: false,
1848            spawned_at_ms: None,
1849            spawned_from: None,
1850            launch_spec_digest: None,
1851            spawned_file_identity: None,
1852            process_start_time: None,
1853            deliberate_severance: None,
1854            last_exit: None,
1855            drain_disposition_detail: None,
1856            health: ModuleHealthStatus::default(),
1857            last_probe_evidence: None,
1858            in_alternate_slot: false,
1859            draining_to_replace: false,
1860            configuration_updated_since_spawn: false,
1861        }
1862    }
1863}
1864
1865type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1866
1867type SpawnSubscriberKey = (ConnectionId, u64);
1868
1869#[derive(Debug)]
1870struct SpawnSubscriber {
1871    version: u8,
1872    frames: mpsc::Sender<Frame>,
1873    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1874    /// from which event. The full frame channel cannot carry that news, so it
1875    /// travels beside it; see `SpawnEventFeed::subscribe`.
1876    lagged: Option<oneshot::Sender<SpawnCursor>>,
1877}
1878
1879#[derive(Debug)]
1880struct SpawnEventState {
1881    daemon_incarnation: String,
1882    seq: u64,
1883    capacity: usize,
1884    live: HashMap<String, LiveSpawn>,
1885    generations: HashMap<String, u64>,
1886    events: VecDeque<SpawnEvent>,
1887    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1888    process_changes: HashMap<String, tokio::sync::watch::Sender<u64>>,
1889}
1890
1891impl Default for SpawnEventState {
1892    fn default() -> Self {
1893        Self {
1894            daemon_incarnation: "unconfigured".to_string(),
1895            seq: 0,
1896            capacity: SPAWN_EVENT_RING_CAPACITY,
1897            live: HashMap::new(),
1898            generations: HashMap::new(),
1899            events: VecDeque::new(),
1900            subscribers: HashMap::new(),
1901            process_changes: HashMap::new(),
1902        }
1903    }
1904}
1905
1906#[derive(Debug, Clone, Default)]
1907struct SpawnEventFeed(
1908    Arc<Mutex<SpawnEventState>>,
1909    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1910);
1911
1912#[derive(Debug, Clone, PartialEq, Eq)]
1913pub(crate) enum SpawnSubscribeRefusal {
1914    ForeignIncarnation { current: String },
1915    TooOld { oldest: SpawnCursor },
1916    Frame(String),
1917}
1918
1919impl SpawnEventFeed {
1920    fn configure_incarnation(&self, daemon_incarnation: String) {
1921        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1922        state.daemon_incarnation = daemon_incarnation;
1923        state.seq = 0;
1924        state.live.clear();
1925        state.generations.clear();
1926        state.events.clear();
1927        state.subscribers.clear();
1928        for changed in state.process_changes.values() {
1929            changed.send_replace(0);
1930        }
1931    }
1932
1933    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1934        SpawnCursor {
1935            daemon_incarnation: state.daemon_incarnation.clone(),
1936            seq: state.seq,
1937        }
1938    }
1939
1940    fn snapshot(&self) -> SpawnSnapshot {
1941        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1942        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1943        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1944        SpawnSnapshot {
1945            cursor: Self::cursor(&state),
1946            ring_bound: state.capacity as u64,
1947            live,
1948        }
1949    }
1950
1951    fn allocate_generation_locked(state: &mut SpawnEventState, module_id: &str) -> u64 {
1952        let generation = state
1953            .generations
1954            .get(module_id)
1955            .copied()
1956            .unwrap_or(0)
1957            .checked_add(1)
1958            .expect("spawn generation exhausted");
1959        state.generations.insert(module_id.to_string(), generation);
1960        generation
1961    }
1962
1963    /// Allocate the spawn generation of a launch that is about to start, with
1964    /// the daemon incarnation it belongs to. The number names the launch from
1965    /// this moment, before its process runs; a launch that never runs keeps
1966    /// its number, so the generations a subscriber sees may skip.
1967    fn allocate_generation(&self, module_id: &str) -> (String, u64) {
1968        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1969        let generation = Self::allocate_generation_locked(&mut state, module_id);
1970        (state.daemon_incarnation.clone(), generation)
1971    }
1972
1973    /// Publish a running process. `allocated` is the generation its launch
1974    /// was given before it started; a launch without one (a process with no
1975    /// registration channel) is numbered here.
1976    fn emit_spawned(
1977        &self,
1978        module_id: &str,
1979        allocated: Option<u64>,
1980        pid: u32,
1981        spawned_at_ms: u64,
1982    ) -> u64 {
1983        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1984        let generation = match allocated {
1985            Some(generation) => generation,
1986            None => Self::allocate_generation_locked(&mut state, module_id),
1987        };
1988        let live = LiveSpawn {
1989            module_id: module_id.to_string(),
1990            spawn_generation: generation,
1991            pid,
1992            spawned_at_ms,
1993        };
1994        state.live.insert(module_id.to_string(), live);
1995        Self::emit_locked(
1996            &mut state,
1997            SpawnEventKind::Spawned,
1998            module_id.to_string(),
1999            generation,
2000            pid,
2001            None,
2002            None,
2003        );
2004        generation
2005    }
2006
2007    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
2008        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2009        let Some(live) = state.live.remove(module_id) else {
2010            warn!(
2011                module_id,
2012                "terminal record had no live spawn event identity"
2013            );
2014            return;
2015        };
2016        Self::emit_locked(
2017            &mut state,
2018            SpawnEventKind::Exited,
2019            module_id.to_string(),
2020            live.spawn_generation,
2021            live.pid,
2022            exit_code,
2023            exit_signal,
2024        );
2025    }
2026
2027    /// Report the exit of a process that a swap has already replaced.
2028    ///
2029    /// `emit_exited` removes the module's live entry, which after a swap's
2030    /// cutover describes the promoted candidate, not the old process now
2031    /// exiting. This emits the old generation's exit and leaves the live entry
2032    /// alone unless it still names that generation.
2033    fn emit_superseded_exited(
2034        &self,
2035        module_id: &str,
2036        spawn_generation: u64,
2037        pid: u32,
2038        exit_code: Option<i32>,
2039        exit_signal: Option<i32>,
2040    ) {
2041        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2042        if state
2043            .live
2044            .get(module_id)
2045            .is_some_and(|live| live.spawn_generation == spawn_generation)
2046        {
2047            state.live.remove(module_id);
2048        }
2049        Self::emit_locked(
2050            &mut state,
2051            SpawnEventKind::Exited,
2052            module_id.to_string(),
2053            spawn_generation,
2054            pid,
2055            exit_code,
2056            exit_signal,
2057        );
2058    }
2059
2060    #[allow(clippy::too_many_arguments)]
2061    fn emit_locked(
2062        state: &mut SpawnEventState,
2063        kind: SpawnEventKind,
2064        module_id: String,
2065        spawn_generation: u64,
2066        pid: u32,
2067        exit_code: Option<i32>,
2068        exit_signal: Option<i32>,
2069    ) {
2070        state.seq = state
2071            .seq
2072            .checked_add(1)
2073            .expect("spawn event sequence exhausted");
2074        let event = SpawnEvent {
2075            cursor: Self::cursor(state),
2076            kind,
2077            module_id,
2078            spawn_generation,
2079            pid,
2080            exit_code,
2081            exit_signal,
2082        };
2083        state
2084            .process_changes
2085            .retain(|_, changed| changed.receiver_count() > 0);
2086        if let Some(changed) = state.process_changes.get(&event.module_id) {
2087            changed.send_replace(state.seq);
2088        }
2089        state.events.push_back(event.clone());
2090        while state.events.len() > state.capacity {
2091            state.events.pop_front();
2092        }
2093        let body = match serde_json::to_vec(&event) {
2094            Ok(body) => body,
2095            Err(error) => {
2096                error!(%error, "failed to serialize supervisor spawn event");
2097                return;
2098            }
2099        };
2100        state.subscribers.retain(|(connection_id, corr), subscriber| {
2101            let frame = Frame::build_with_version(
2102                subscriber.version,
2103                FrameType::StreamData,
2104                control_flags(),
2105                0,
2106                0,
2107                *corr,
2108                body.clone(),
2109            );
2110            match frame {
2111                Ok(frame) => {
2112                    if subscriber.frames.try_send(frame).is_ok() {
2113                        true
2114                    } else {
2115                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
2116                        if let Some(lagged) = subscriber.lagged.take() {
2117                            let _ = lagged.send(event.cursor.clone());
2118                        }
2119                        false
2120                    }
2121                }
2122                Err(error) => {
2123                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
2124                    false
2125                }
2126            }
2127        });
2128    }
2129
2130    fn subscribe(
2131        &self,
2132        connection_id: ConnectionId,
2133        corr: u64,
2134        version: u8,
2135        since: Option<SpawnCursor>,
2136        sink: FrameSink,
2137    ) -> Result<(), SpawnSubscribeRefusal> {
2138        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
2139        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
2140        {
2141            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2142            let replay = if let Some(since) = since {
2143                if since.daemon_incarnation != state.daemon_incarnation {
2144                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
2145                        current: state.daemon_incarnation.clone(),
2146                    });
2147                }
2148                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
2149                    if since.seq < oldest.seq.saturating_sub(1) {
2150                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
2151                    }
2152                }
2153                state
2154                    .events
2155                    .iter()
2156                    .filter(|event| event.cursor.seq > since.seq)
2157                    .cloned()
2158                    .collect::<Vec<_>>()
2159            } else {
2160                Vec::new()
2161            };
2162            for event in replay {
2163                let body = serde_json::to_vec(&event)
2164                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2165                let frame = Frame::build_with_version(
2166                    version,
2167                    FrameType::StreamData,
2168                    control_flags(),
2169                    0,
2170                    0,
2171                    corr,
2172                    body,
2173                )
2174                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2175                frames
2176                    .try_send(frame)
2177                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2178            }
2179            state.subscribers.insert(
2180                (connection_id, corr),
2181                SpawnSubscriber {
2182                    version,
2183                    frames,
2184                    lagged: Some(lagged),
2185                },
2186            );
2187        }
2188        // The lagged terminal is sent here, by the forwarder, rather than by
2189        // the emitter: at the moment of the drop the subscriber's own channel
2190        // is full, and writing to the connection sink directly from the emitter
2191        // would put the Error AHEAD of the events still queued in that channel
2192        // (and the emitter holds the feed lock, so it cannot await the sink).
2193        // Dropping the subscriber drops the only sender, so `recv` drains every
2194        // queued event and then returns `None`; only then is the Error sent, so
2195        // the client sees each event it can keep, then the reason it was cut.
2196        // Cancel and connection removal drop the oneshot unsent, so they end
2197        // the stream with no Error.
2198        tokio::spawn(async move {
2199            while let Some(frame) = receiver.recv().await {
2200                if sink.send(frame).await.is_err() {
2201                    return;
2202                }
2203            }
2204            let Ok(first_undelivered) = lagged_rx.try_recv() else {
2205                return;
2206            };
2207            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
2208                Ok(frame) => {
2209                    let _ = sink.send(frame).await;
2210                }
2211                Err(error) => {
2212                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
2213                }
2214            }
2215        });
2216        Ok(())
2217    }
2218
2219    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
2220        let Some(subscriber) = self
2221            .0
2222            .lock()
2223            .unwrap_or_else(|p| p.into_inner())
2224            .subscribers
2225            .remove(&(connection_id, corr))
2226        else {
2227            return false;
2228        };
2229        if let Ok(frame) = Frame::build_with_version(
2230            subscriber.version,
2231            FrameType::StreamEnd,
2232            control_flags(),
2233            0,
2234            0,
2235            corr,
2236            Vec::new(),
2237        ) {
2238            tokio::spawn(async move {
2239                let _ = subscriber.frames.send(frame).await;
2240            });
2241        }
2242        true
2243    }
2244
2245    fn remove_connection(&self, connection_id: ConnectionId) {
2246        self.0
2247            .lock()
2248            .unwrap_or_else(|p| p.into_inner())
2249            .subscribers
2250            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
2251    }
2252
2253    #[cfg(any(test, feature = "test-support"))]
2254    fn set_capacity(&self, capacity: usize) {
2255        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
2256    }
2257
2258    #[cfg(any(test, feature = "test-support"))]
2259    fn subscriber_count(&self) -> usize {
2260        self.0
2261            .lock()
2262            .unwrap_or_else(|p| p.into_inner())
2263            .subscribers
2264            .len()
2265    }
2266}
2267
2268/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
2269/// The terminal Error a lagged spawn subscriber receives after its queued events.
2270fn spawn_subscriber_lagged_frame(
2271    version: u8,
2272    corr: u64,
2273    first_undelivered: SpawnCursor,
2274) -> Result<Frame, String> {
2275    // The daemon authors this terminal Error, so it carries the daemon-origin bit.
2276    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
2277        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
2278        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
2279            .to_string(),
2280        detail: Some(serde_json::json!({
2281            "first_undelivered_cursor": first_undelivered
2282        })),
2283    })
2284    .map_err(|error| error.to_string())?;
2285    Frame::build_with_version(
2286        version,
2287        FrameType::Error,
2288        crate::router::daemon_error_flags(),
2289        0,
2290        0,
2291        corr,
2292        body,
2293    )
2294    .map_err(|error| error.to_string())
2295}
2296
2297pub trait ModuleProcessLiveness: Send + Sync {
2298    fn process_live(&self, module_id: &str) -> Option<bool>;
2299
2300    /// Whether the supervisor is replacing this module's process right now: an
2301    /// operator restart or reload, a health restart, or a crash respawn whose
2302    /// backoff is running. A module in that state is not live, but a consumer
2303    /// refused now should retry shortly rather than treat the target as gone.
2304    /// Stopped, failed, and disabled modules are not replacing.
2305    fn process_replacing(&self, _module_id: &str) -> bool {
2306        false
2307    }
2308}
2309
2310/// Shared process-liveness registry keyed by supervised `module_id`.
2311#[derive(Debug, Clone, Default)]
2312pub struct SupervisorProcessLiveness {
2313    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
2314}
2315
2316impl SupervisorProcessLiveness {
2317    pub fn new() -> Self {
2318        Self::default()
2319    }
2320
2321    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
2322        let mut snapshots = self
2323            .snapshots
2324            .lock()
2325            .unwrap_or_else(|poisoned| poisoned.into_inner());
2326        snapshots.insert(module_id, snapshot);
2327    }
2328
2329    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
2330        let mut snapshots = self
2331            .snapshots
2332            .lock()
2333            .unwrap_or_else(|poisoned| poisoned.into_inner());
2334        let is_current = snapshots
2335            .get(module_id)
2336            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
2337            .unwrap_or(false);
2338        if is_current {
2339            snapshots.remove(module_id);
2340        }
2341    }
2342}
2343
2344impl ModuleProcessLiveness for SupervisorProcessLiveness {
2345    fn process_live(&self, module_id: &str) -> Option<bool> {
2346        let snapshot = {
2347            let snapshots = self
2348                .snapshots
2349                .lock()
2350                .unwrap_or_else(|poisoned| poisoned.into_inner());
2351            snapshots.get(module_id).cloned()
2352        }?;
2353        let snapshot = snapshot
2354            .lock()
2355            .unwrap_or_else(|poisoned| poisoned.into_inner());
2356        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
2357    }
2358
2359    fn process_replacing(&self, module_id: &str) -> bool {
2360        let Some(snapshot) = self
2361            .snapshots
2362            .lock()
2363            .unwrap_or_else(|poisoned| poisoned.into_inner())
2364            .get(module_id)
2365            .cloned()
2366        else {
2367            return false;
2368        };
2369        let snapshot = snapshot
2370            .lock()
2371            .unwrap_or_else(|poisoned| poisoned.into_inner());
2372        snapshot.enabled
2373            && match snapshot.state {
2374                ModuleState::Restarting => true,
2375                ModuleState::Draining => snapshot.draining_to_replace,
2376                ModuleState::Starting
2377                | ModuleState::Running
2378                | ModuleState::Unresponsive
2379                | ModuleState::Stopped
2380                | ModuleState::Failed
2381                | ModuleState::Disabled => false,
2382            }
2383    }
2384}
2385
2386#[cfg(test)]
2387#[derive(Debug, Default)]
2388struct ReloadExitRecordGate {
2389    reached: tokio::sync::Notify,
2390    resume: tokio::sync::Notify,
2391}
2392
2393#[derive(Debug, Clone, Copy)]
2394enum RespawnKind {
2395    Spawn,
2396    Reload,
2397}
2398
2399#[derive(Debug, Clone)]
2400struct PendingRespawn {
2401    deadline: Instant,
2402    kind: RespawnKind,
2403    daemon_initiated: bool,
2404    approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
2405}
2406
2407impl PendingRespawn {
2408    fn failure_context(&self, runtime: &SupervisorRuntimeConfig) -> FailureContext {
2409        FailureContext {
2410            approval: self.approval.clone(),
2411            ..runtime.failure_context(self.daemon_initiated)
2412        }
2413    }
2414}
2415
2416type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
2417
2418#[derive(Debug, Clone)]
2419struct SupervisorRuntimeConfig {
2420    registry: Arc<Registry>,
2421    config_delivery: Option<config_delivery::ConfigDelivery>,
2422    /// Retained across an operation's deferred backoff, until its child starts.
2423    recovery_is_daemon: bool,
2424    start_requester: Option<Principal>,
2425    pending_activity_restart: bool,
2426    provenance_probe: ExecutableIdentityProbe,
2427    /// Restart operations hand their backoff to the loop, which keeps serving commands.
2428    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2429    /// A reload acknowledges completion only after its replacement registers.
2430    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2431    restart_policy: RestartPolicy,
2432    register_timeout: Duration,
2433    /// This module's RESOLVED drain budget: per-module config when present,
2434    /// else `default_drain_timeout`.
2435    drain_timeout: Duration,
2436    /// Shared with the status handle so the attested value changes atomically
2437    /// when a rescan updates the running drain policy.
2438    effective_drain_timeout: Arc<Mutex<Duration>>,
2439    /// The supervisor-wide fallback, kept so a configuration update that
2440    /// REMOVES the per-module override can re-resolve to it.
2441    default_drain_timeout: Duration,
2442    health: HealthConfig,
2443    host_health: HostHealth,
2444    connection_file_path: Option<PathBuf>,
2445    capture_logs_dir: Option<PathBuf>,
2446    forwarding: Option<Arc<ForwardingTable>>,
2447    /// The shared handle, so every spawn path (initial, restart, reload) records the
2448    /// reserved-module launch nonce the HELLO verifier checks against.
2449    supervisor_handle: Option<SupervisorHandle>,
2450    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
2451    /// status queries.
2452    ///
2453    /// One ring per module, held across every respawn. The lines explaining an exit
2454    /// are written BEFORE that exit, so a ring recreated per process would be empty
2455    /// exactly when it is asked for.
2456    stderr_ring: Arc<Mutex<StderrRing>>,
2457    terminal_ring: Arc<Mutex<TerminalRing>>,
2458    spawn_events: SpawnEventFeed,
2459    child_roster: ChildRoster,
2460    #[cfg(target_os = "linux")]
2461    cgroup_placement: Option<subc_cgroup::Placement>,
2462    #[cfg(test)]
2463    test_seed_stale_facts_before_enable_spawn: bool,
2464    #[cfg(test)]
2465    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2466}
2467
2468impl SupervisorRuntimeConfig {
2469    fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2470        FailureContext {
2471            daemon_initiated,
2472            approval: None,
2473            shutdown: self.child_roster.shutdown_flag(),
2474        }
2475    }
2476}
2477
2478#[derive(Debug, Clone, PartialEq, Eq)]
2479struct SupervisedConfiguration {
2480    spec: ModuleSpec,
2481    enabled: bool,
2482    health: HealthConfig,
2483    register_timeout: Duration,
2484    restart_policy: RestartPolicy,
2485}
2486
2487/// Shared daemon lookup table for supervised module handles.
2488///
2489/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
2490/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
2491/// launch nonces recorded at spawn are checked by the same daemon instance.
2492#[derive(Debug, Clone, Default)]
2493pub struct SupervisorHandle {
2494    #[cfg(test)]
2495    resource_commit_pause: Arc<Mutex<Option<resource_launch::ResourceCommitPause>>>,
2496    #[cfg(test)]
2497    registration_test_channels: CapturedRegistrationChannels,
2498    registration_service: Arc<Mutex<Option<registration::RegistrationService>>>,
2499    registration_connections: Arc<Mutex<HashMap<ConnectionId, Arc<RegistrationSpawn>>>>,
2500    registration_launches: Arc<Mutex<Vec<std::sync::Weak<RegistrationSpawn>>>>,
2501    gate_runtime: Arc<Mutex<Option<std::sync::Weak<crate::config_gate::runtime::RuntimeOwner>>>>,
2502    program_bindings: program_binding::ProgramBindings,
2503    #[cfg(test)]
2504    identity_sample_pause: Arc<Mutex<Option<IdentitySamplePause>>>,
2505    #[cfg(all(test, unix))]
2506    plain_identity_policy_pause: Arc<Mutex<Option<(String, IdentitySamplePause)>>>,
2507    host_health: HostHealth,
2508    restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2509    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2510    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2511    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2512    /// Module ids the supervisor has taken on. An id is added BEFORE the
2513    /// module's first process is spawned and removed only when the module
2514    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
2515    /// the keys of `modules`.
2516    ///
2517    /// `modules` cannot answer "is this module configured?" on its own: a
2518    /// [`SupervisedModule`] only exists once its process has been spawned, and
2519    /// a fast child can connect, register, sync its scopes and ask about them
2520    /// before the supervisor has inserted it. Answering "not configured" in that
2521    /// gap makes scope admission refuse with the terminal "will never sync"
2522    /// instead of the retryable "has not synced yet".
2523    configured_ids: Arc<Mutex<HashSet<String>>>,
2524    spawn_events: SpawnEventFeed,
2525    /// The current expected launch nonce for each reserved module_id. Set when the
2526    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
2527    /// non-reserved module never has an entry here and is never nonce-checked.
2528    /// Reserved module ids and the nonce that authorizes their next HELLO.
2529    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
2530    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
2531    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
2532    /// had NO entry and admitted anyone: the reservation protected the nonce
2533    /// holder, not the NAME (found live by CKCRED's canary probe registering
2534    /// against a reserved scratch id).
2535    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2536    /// Reservations installed only to prevent HELLO for a protocol:none process.
2537    /// Replacement by a subc process releases them; explicitly reserved ids are
2538    /// never placed in this set and still require an operator release.
2539    implicit_protocol_none: Arc<Mutex<HashSet<String>>>,
2540    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2541    ///
2542    /// This is deliberately in-memory only: subc is state-free across daemon
2543    /// restarts, and the tombstone only explains the hours-after-removal window
2544    /// while this executing daemon is still alive. Do not persist it in a store.
2545    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2546    /// The current launch nonce for every supervised spawn. This is separate from
2547    /// reserved_nonces because consumer route.open attestation applies to all spawned
2548    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2549    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2550    /// The launch each live launch nonce was minted for, keyed by nonce, with
2551    /// the launch's module id, so a nonce-attested consumer or registration
2552    /// resolves to that launch's authority record. An entry whose launch has
2553    /// ended stays while its nonce still authorizes, so a nonce whose record
2554    /// is gone is told apart from one that never had a record.
2555    nonce_launches: Arc<Mutex<NonceLaunches>>,
2556    /// Reserved namespace prefixes mapped to the supervised owner module whose
2557    /// current spawn nonce authorizes HELLO claims below the prefix.
2558    ///
2559    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2560    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2561    /// accidental collisions and lower-trust processes from squatting protected
2562    /// namespaces.
2563    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2564    /// Blue/green swaps in progress, by module id. An entry exists from just
2565    /// before the candidate process is spawned until the swap has failed, or
2566    /// has cut over and the old process is gone. While it exists, HELLO for the
2567    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2568    /// consumer attestation accepts both processes' nonces.
2569    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2570    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2571    promotion_observer: PromotionObserverSlot,
2572    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2573    /// this daemon-wide ordering, a rescan could retire or update a module while a
2574    /// concurrent reload still held its old handle and launch specification.
2575    operation_lock: Arc<AsyncMutex<()>>,
2576}
2577
2578#[cfg(test)]
2579#[derive(Debug)]
2580struct IdentitySamplePause {
2581    reached: std::sync::mpsc::Sender<()>,
2582    release: std::sync::mpsc::Receiver<()>,
2583}
2584
2585/// Told when a swap has promoted its candidate to be the module's active
2586/// registration.
2587///
2588/// An ordinary HELLO runs the control plane's registration side effects (the
2589/// capability cache, the deny census, the requirement recompute) as it
2590/// registers. A swap candidate's HELLO does not, because it is not routable;
2591/// promotion is when those must run instead, and promotion happens in the
2592/// supervisor, which has no other way into the control handler.
2593pub(crate) trait SwapPromotionObserver: Send + Sync {
2594    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2595}
2596
2597/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2598/// control handler) owns this handle, so a strong reference back would be a
2599/// cycle that keeps both alive.
2600#[derive(Clone, Default)]
2601struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2602
2603impl fmt::Debug for PromotionObserverSlot {
2604    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2605        f.write_str("PromotionObserverSlot")
2606    }
2607}
2608
2609/// The nonces of one open swap.
2610#[derive(Debug, Clone)]
2611struct OpenSwap {
2612    /// Legacy candidate token. A gated candidate has an empty lifecycle
2613    /// placeholder instead; only its own channel admits its HELLO.
2614    candidate_nonce: String,
2615    /// Incumbent token accepted before cutover, then cleared by promotion even
2616    /// while the incumbent's process is still draining.
2617    incumbent_nonce: Option<String>,
2618    /// Set once a HELLO has been admitted with the swap token, so the token
2619    /// admits one registration and cannot be replayed after cutover empties
2620    /// the candidate slot.
2621    candidate_admitted: bool,
2622}
2623
2624/// What the swap gate says about a HELLO. See
2625/// [`SupervisorHandle::swap_hello_admission`].
2626#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2627pub(crate) enum SwapHelloAdmission {
2628    /// No swap is open for the id (or the HELLO carries the incumbent's own
2629    /// nonce); the ordinary gates decide.
2630    NotSwapping,
2631    /// The HELLO carries the swap token: register it into the candidate slot.
2632    Candidate,
2633    /// A swap is open and the HELLO carries a nonce the supervisor did not
2634    /// mint for this id, no nonce, or a token already used.
2635    Refused,
2636    /// The id belongs to a supervised `protocol: none` module. Such a module
2637    /// never registers, so no HELLO may claim its id, with or without a nonce.
2638    ProtocolNone,
2639    /// The id is gated. Its supervised child registers only over the
2640    /// registration channel it inherited at launch, never by HELLO alone.
2641    ChannelRequired,
2642}
2643
2644#[derive(Debug, Clone, PartialEq, Eq)]
2645pub(crate) enum ReservedHelloRejection {
2646    Exact {
2647        module_id: String,
2648    },
2649    Prefix {
2650        prefix: String,
2651        owner_module_id: String,
2652    },
2653}
2654
2655/// Identity facts established by the reserved-module policy for one HELLO.
2656#[derive(Debug, Default)]
2657pub(crate) struct ReservedHelloVerification {
2658    pub(crate) nonce_verified: bool,
2659    pub(crate) prefix_owner: Option<String>,
2660}
2661
2662#[derive(Debug)]
2663struct DaemonRestartIntent {
2664    expires: Option<Duration>,
2665    window: Duration,
2666}
2667
2668impl Default for DaemonRestartIntent {
2669    fn default() -> Self {
2670        Self {
2671            expires: None,
2672            window: Duration::from_secs(120),
2673        }
2674    }
2675}
2676
2677impl SupervisorHandle {
2678    pub(crate) fn module_is_gated(&self, module: &str) -> bool {
2679        let runtime = {
2680            let Ok(runtime) = self.gate_runtime.lock() else {
2681                return true;
2682            };
2683            runtime.clone()
2684        };
2685        runtime.is_some_and(|runtime| {
2686            runtime
2687                .upgrade()
2688                .is_none_or(|owner| owner.module_is_gated(module))
2689        })
2690    }
2691    pub(crate) fn announce_daemon_restart(&self) -> u64 {
2692        let mut intent = self
2693            .restart_intent
2694            .lock()
2695            .unwrap_or_else(|p| p.into_inner());
2696        intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2697        unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2698    }
2699
2700    #[cfg(feature = "test-support")]
2701    pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2702        self.restart_intent
2703            .lock()
2704            .unwrap_or_else(|p| p.into_inner())
2705            .window = window;
2706    }
2707
2708    #[cfg(unix)]
2709    fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2710        let expiry = self
2711            .restart_intent
2712            .lock()
2713            .unwrap_or_else(|p| p.into_inner())
2714            .expires
2715            .take();
2716        if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2717            RouteCloseReason::Restart
2718        } else {
2719            RouteCloseReason::Disable
2720        }
2721    }
2722
2723    pub fn new() -> Self {
2724        Self::default()
2725    }
2726
2727    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2728        self.spawn_events.snapshot()
2729    }
2730
2731    /// Subscribe before reading the bound process. A coalesced wake is enough:
2732    /// readers compare its exact generation and PID, not the last event kind.
2733    pub(crate) fn subscribe_process_changes(
2734        &self,
2735        module: &str,
2736    ) -> tokio::sync::watch::Receiver<u64> {
2737        let mut state = self
2738            .spawn_events
2739            .0
2740            .lock()
2741            .unwrap_or_else(|p| p.into_inner());
2742        state
2743            .process_changes
2744            .retain(|_, changed| changed.receiver_count() > 0);
2745        let seq = state.seq;
2746        state
2747            .process_changes
2748            .entry(module.into())
2749            .or_insert_with(|| tokio::sync::watch::channel(seq).0)
2750            .subscribe()
2751    }
2752
2753    pub(crate) fn current_spawn(&self, module: &str) -> Option<LiveSpawn> {
2754        self.spawn_events
2755            .0
2756            .lock()
2757            .unwrap_or_else(|p| p.into_inner())
2758            .live
2759            .get(module)
2760            .cloned()
2761    }
2762
2763    pub(crate) fn subscribe_spawns(
2764        &self,
2765        connection_id: ConnectionId,
2766        corr: u64,
2767        version: u8,
2768        since: Option<SpawnCursor>,
2769        sink: FrameSink,
2770    ) -> Result<(), SpawnSubscribeRefusal> {
2771        self.spawn_events
2772            .subscribe(connection_id, corr, version, since, sink)
2773    }
2774
2775    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2776        self.spawn_events.cancel(connection_id, corr)
2777    }
2778
2779    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2780        self.spawn_events.remove_connection(connection_id);
2781    }
2782
2783    #[cfg(any(test, feature = "test-support"))]
2784    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2785        assert!(capacity > 0, "spawn event capacity must be non-zero");
2786        self.spawn_events.set_capacity(capacity);
2787    }
2788
2789    #[cfg(any(test, feature = "test-support"))]
2790    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2791        self.spawn_events.subscriber_count()
2792    }
2793
2794    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2795    /// a respawn invalidates stale consumer identities.
2796    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2797        self.spawn_nonces
2798            .lock()
2799            .unwrap_or_else(|poisoned| poisoned.into_inner())
2800            .insert(module_id.to_string(), nonce);
2801        self.refresh_program_binding_nonces(module_id);
2802    }
2803
2804    /// Record the launch nonce expected from the next HELLO for a reserved module,
2805    /// replacing any prior nonce (a respawn invalidates the previous one).
2806    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2807        {
2808            let mut reserved = self
2809                .reserved_nonces
2810                .lock()
2811                .unwrap_or_else(|p| p.into_inner());
2812            self.implicit_protocol_none
2813                .lock()
2814                .unwrap_or_else(|p| p.into_inner())
2815                .remove(module_id);
2816            reserved.insert(module_id.to_string(), Some(nonce));
2817        }
2818        self.refresh_program_binding_nonces(module_id);
2819    }
2820
2821    /// Record namespace prefixes owned by a supervised module.
2822    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2823        let mut owners = self
2824            .reserved_prefix_owners
2825            .lock()
2826            .unwrap_or_else(|poisoned| poisoned.into_inner());
2827        owners.retain(|_, owner| owner != owner_module_id);
2828        for prefix in prefixes {
2829            owners.insert(prefix.clone(), owner_module_id.to_string());
2830        }
2831    }
2832
2833    /// The launch nonce most recently minted for a module's spawn, if any.
2834    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2835        if self.module_is_gated(module_id) {
2836            return None;
2837        }
2838        self.spawn_nonces
2839            .lock()
2840            .unwrap_or_else(|poisoned| poisoned.into_inner())
2841            .get(module_id)
2842            .cloned()
2843    }
2844
2845    #[cfg(all(test, unix))]
2846    pub(crate) fn legacy_nonce_present_for_test(&self, module: &str) -> bool {
2847        self.spawn_nonces.lock().unwrap().contains_key(module)
2848    }
2849
2850    pub(crate) fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2851        // Prefix ownership follows staged configuration, not the running
2852        // child's launch settings. A respawn may still carry an older spec.
2853        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2854        let protocol = self
2855            .get(&spec.module_id)
2856            .and_then(|module| {
2857                let snapshot = module.inner.snapshot.lock().ok()?;
2858                snapshot
2859                    .process_alive
2860                    .then_some(snapshot.spawned_protocol.unwrap_or(spec.protocol))
2861            })
2862            .unwrap_or(spec.protocol);
2863        self.apply_identity_policy(spec, protocol);
2864    }
2865
2866    #[cfg(all(test, unix))]
2867    pub(crate) fn pause_plain_identity_policy_for_test(
2868        &self,
2869        module: &str,
2870    ) -> (std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>) {
2871        let (reached, waiting) = std::sync::mpsc::channel();
2872        let (release, paused) = std::sync::mpsc::channel();
2873        *self.plain_identity_policy_pause.lock().unwrap() = Some((
2874            module.to_owned(),
2875            IdentitySamplePause {
2876                reached,
2877                release: paused,
2878            },
2879        ));
2880        (waiting, release)
2881    }
2882
2883    fn apply_identity_policy(&self, spec: &ModuleSpec, protocol: ModuleProtocol) {
2884        let channel_only = self.module_is_gated(&spec.module_id);
2885        let mut current_nonces = self
2886            .spawn_nonces
2887            .lock()
2888            .unwrap_or_else(|poisoned| poisoned.into_inner());
2889        if protocol == ModuleProtocol::None || channel_only {
2890            current_nonces.remove(&spec.module_id);
2891        }
2892        let spawn_nonce = current_nonces.get(&spec.module_id).cloned();
2893        #[cfg(test)]
2894        if let Some(pause) = self.identity_sample_pause.lock().unwrap().take() {
2895            pause.reached.send(()).unwrap();
2896            pause.release.recv().unwrap();
2897        }
2898        let mut reserved_nonces = self
2899            .reserved_nonces
2900            .lock()
2901            .unwrap_or_else(|poisoned| poisoned.into_inner());
2902        let mut implicit = self
2903            .implicit_protocol_none
2904            .lock()
2905            .unwrap_or_else(|p| p.into_inner());
2906        if spec.reserved {
2907            implicit.remove(&spec.module_id);
2908        } else if protocol == ModuleProtocol::None {
2909            if !reserved_nonces.contains_key(&spec.module_id) {
2910                implicit.insert(spec.module_id.clone());
2911            }
2912        } else if implicit.remove(&spec.module_id) {
2913            reserved_nonces.remove(&spec.module_id);
2914        }
2915        if spec.reserved || protocol == ModuleProtocol::None {
2916            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2917            // reserved name whose module has never spawned has no legitimate
2918            // holder, and the entry's absence is what used to leave the name
2919            // open to the first claimant.
2920            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2921        }
2922        drop(implicit);
2923        drop(reserved_nonces);
2924        drop(current_nonces);
2925        self.refresh_program_binding_nonces(&spec.module_id);
2926        // A later unreserved declaration must not silently unreserve an id that
2927        // was retained after its reserved configuration was removed. The explicit
2928        // release ceremony is the only operation that retires that gate.
2929        self.removal_tombstones
2930            .lock()
2931            .unwrap_or_else(|poisoned| poisoned.into_inner())
2932            .remove(&spec.module_id);
2933    }
2934
2935    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2936    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2937    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2938    /// with no matching prefix are always authorized.
2939    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2940        self.reserved_hello_verification(module_id, presented)
2941            .is_ok()
2942    }
2943
2944    /// Whether HELLO passed a nonce check that was actually required by reserved
2945    /// module policy, or the refusal explaining why a required check failed.
2946    pub(crate) fn reserved_hello_verification(
2947        &self,
2948        module_id: &str,
2949        presented: Option<&str>,
2950    ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2951        if self.module_is_gated(module_id) {
2952            return Err(ReservedHelloRejection::Exact {
2953                module_id: module_id.into(),
2954            });
2955        }
2956        self.hello_verification(module_id, presented, false)
2957    }
2958
2959    pub(crate) fn channel_hello_verification(
2960        &self,
2961        module_id: &str,
2962    ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2963        if self
2964            .get(module_id)
2965            .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
2966        {
2967            return Err(ReservedHelloRejection::Exact {
2968                module_id: module_id.into(),
2969            });
2970        }
2971        let mut verification = self.hello_verification(module_id, None, true)?;
2972        verification.nonce_verified = true;
2973        Ok(verification)
2974    }
2975
2976    pub(crate) fn prefix_claim_authorized(&self, owner: &str, module_id: &str) -> bool {
2977        self.prefix_claim_publication(owner, module_id).is_some()
2978    }
2979
2980    #[cfg(all(test, unix))]
2981    pub(crate) fn prefix_publication_locked_for_test(&self) -> bool {
2982        matches!(
2983            self.reserved_prefix_owners.try_lock(),
2984            Err(std::sync::TryLockError::WouldBlock)
2985        )
2986    }
2987
2988    // Keep this guard through registry/forwarding publication. Rescans replace
2989    // prefix owners without stopping their launches, so the spawn guard alone
2990    // cannot fence a reservation transfer.
2991    pub(crate) fn prefix_claim_publication(
2992        &self,
2993        owner: &str,
2994        module_id: &str,
2995    ) -> Option<std::sync::MutexGuard<'_, HashMap<String, String>>> {
2996        if subc_protocol::manifest::validate_module_id(module_id).is_err()
2997            || self.module_is_gated(module_id)
2998            || self
2999                .reserved_nonces
3000                .lock()
3001                .unwrap_or_else(|p| p.into_inner())
3002                .contains_key(module_id)
3003        {
3004            return None;
3005        }
3006        let owners = self
3007            .reserved_prefix_owners
3008            .lock()
3009            .unwrap_or_else(|p| p.into_inner());
3010        let authorized = owners
3011            .iter()
3012            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
3013            .max_by_key(|(prefix, _)| prefix.len())
3014            .is_some_and(|(prefix, holder)| holder == owner && module_id.len() > prefix.len());
3015        authorized.then_some(owners)
3016    }
3017
3018    fn hello_verification(
3019        &self,
3020        module_id: &str,
3021        presented: Option<&str>,
3022        channel: bool,
3023    ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
3024        let nonces = self
3025            .reserved_nonces
3026            .lock()
3027            .unwrap_or_else(|poisoned| poisoned.into_inner());
3028        if let Some(expected) = nonces.get(module_id) {
3029            // `None` = reserved with no legitimate holder: refuse every
3030            // presentation, because no process can hold a nonce that was never
3031            // minted. Only a real minted nonce admits, in constant time.
3032            let authorized = channel
3033                || match expected {
3034                    Some(expected) => presented.is_some_and(|p| {
3035                        !p.is_empty() && constant_time_eq(expected.as_bytes(), p.as_bytes())
3036                    }),
3037                    None => false,
3038                };
3039            if authorized {
3040                return Ok(ReservedHelloVerification {
3041                    nonce_verified: true,
3042                    prefix_owner: None,
3043                });
3044            }
3045            return Err(ReservedHelloRejection::Exact {
3046                module_id: module_id.to_string(),
3047            });
3048        }
3049        drop(nonces);
3050
3051        let matched_prefix = self
3052            .reserved_prefix_owners
3053            .lock()
3054            .unwrap_or_else(|poisoned| poisoned.into_inner())
3055            .iter()
3056            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
3057            .max_by_key(|(prefix, _)| prefix.len())
3058            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
3059        let Some((prefix, owner_module_id)) = matched_prefix else {
3060            return Ok(ReservedHelloVerification::default());
3061        };
3062
3063        if module_id.len() == prefix.len()
3064            || (!channel
3065                && (self.module_is_gated(&owner_module_id) || self.protocol_none(&owner_module_id)))
3066        {
3067            return Err(ReservedHelloRejection::Prefix {
3068                prefix,
3069                owner_module_id,
3070            });
3071        }
3072
3073        let authorized = if channel {
3074            owner_module_id == module_id
3075        } else {
3076            presented.is_some_and(|presented| {
3077                self.spawn_nonces
3078                .lock()
3079                .unwrap_or_else(|poisoned| poisoned.into_inner())
3080                .get(&owner_module_id)
3081                .is_some_and(|expected| !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes()))
3082                // While the owner is being swapped, children started by
3083                // either of its two processes hold that process's nonce.
3084                || self.swap_nonce_matches(&owner_module_id, presented)
3085            })
3086        };
3087        if authorized {
3088            Ok(ReservedHelloVerification {
3089                nonce_verified: true,
3090                prefix_owner: (owner_module_id != module_id).then_some(owner_module_id),
3091            })
3092        } else {
3093            Err(ReservedHelloRejection::Prefix {
3094                prefix,
3095                owner_module_id,
3096            })
3097        }
3098    }
3099
3100    /// Whether a consumer connection proved it came from a daemon-spawned module.
3101    ///
3102    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
3103    /// accepted only for module ids the supervisor has spawned.
3104    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
3105        if self.module_is_gated(module_id) || self.protocol_none(module_id) {
3106            return false;
3107        }
3108        let nonces = self
3109            .spawn_nonces
3110            .lock()
3111            .unwrap_or_else(|poisoned| poisoned.into_inner());
3112        let current = nonces.get(module_id).is_some_and(|expected| {
3113            !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes())
3114        });
3115        drop(nonces);
3116        // Before cutover both launches may attest. Promotion removes the
3117        // incumbent's token even if its process still needs time to drain.
3118        current || self.swap_nonce_matches(module_id, presented)
3119    }
3120
3121    /// Whether `presented` is either nonce of an open swap for `module_id`.
3122    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
3123        if self.module_is_gated(module_id) || self.protocol_none(module_id) || presented.is_empty()
3124        {
3125            return false;
3126        }
3127        let swaps = self
3128            .swaps
3129            .lock()
3130            .unwrap_or_else(|poisoned| poisoned.into_inner());
3131        swaps.get(module_id).is_some_and(|swap| {
3132            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
3133                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
3134                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
3135                })
3136        })
3137    }
3138
3139    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
3140    /// Called before the candidate process exists.
3141    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
3142        let incumbent_nonce = self
3143            .spawn_nonces
3144            .lock()
3145            .unwrap_or_else(|poisoned| poisoned.into_inner())
3146            .get(module_id)
3147            .cloned();
3148        self.swaps
3149            .lock()
3150            .unwrap_or_else(|poisoned| poisoned.into_inner())
3151            .insert(
3152                module_id.to_string(),
3153                OpenSwap {
3154                    candidate_nonce,
3155                    incumbent_nonce,
3156                    candidate_admitted: false,
3157                },
3158            );
3159        self.refresh_program_binding_nonces(module_id);
3160    }
3161
3162    /// Close the swap for `module_id`, releasing whichever nonce is no longer
3163    /// the module's recorded one.
3164    pub(crate) fn close_swap(&self, module_id: &str) {
3165        self.swaps
3166            .lock()
3167            .unwrap_or_else(|poisoned| poisoned.into_inner())
3168            .remove(module_id);
3169        self.refresh_program_binding_nonces(module_id);
3170    }
3171
3172    /// Install the observer told about swap promotions, replacing any earlier
3173    /// one.
3174    pub(crate) fn set_swap_promotion_observer(
3175        &self,
3176        observer: std::sync::Weak<dyn SwapPromotionObserver>,
3177    ) {
3178        *self
3179            .promotion_observer
3180            .0
3181            .lock()
3182            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
3183    }
3184
3185    /// Tell the installed observer, if it is still alive, that a swap promoted
3186    /// `registration`.
3187    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
3188        let observer = self
3189            .promotion_observer
3190            .0
3191            .lock()
3192            .unwrap_or_else(|poisoned| poisoned.into_inner())
3193            .as_ref()
3194            .and_then(std::sync::Weak::upgrade);
3195        if let Some(observer) = observer {
3196            observer.swap_promoted(registration);
3197        }
3198    }
3199
3200    /// Whether a swap is open for `module_id`.
3201    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
3202        self.swaps
3203            .lock()
3204            .unwrap_or_else(|poisoned| poisoned.into_inner())
3205            .contains_key(module_id)
3206    }
3207
3208    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
3209    /// respawn would, once cutover has made the candidate the module's process.
3210    /// Promotion revokes the incumbent's tokens; the swap record stays open
3211    /// only for lifecycle cleanup while the old process drains.
3212    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
3213        let candidate_nonce = self
3214            .swaps
3215            .lock()
3216            .unwrap_or_else(|poisoned| poisoned.into_inner())
3217            .get_mut(module_id)
3218            .map(|swap| {
3219                swap.incumbent_nonce = None;
3220                swap.candidate_nonce.clone()
3221            });
3222        self.spawn_nonces
3223            .lock()
3224            .unwrap_or_else(|p| p.into_inner())
3225            .remove(module_id);
3226        {
3227            let mut nonces = self
3228                .reserved_nonces
3229                .lock()
3230                .unwrap_or_else(|p| p.into_inner());
3231            if reserved || nonces.contains_key(module_id) {
3232                nonces.insert(module_id.into(), None);
3233            }
3234        }
3235        if self.module_is_gated(module_id) {
3236            return;
3237        }
3238        let Some(nonce) = candidate_nonce.filter(|nonce| !nonce.is_empty()) else {
3239            return;
3240        };
3241        self.set_spawn_nonce(module_id, nonce.clone());
3242        if reserved {
3243            self.set_reserved_nonce(module_id, nonce);
3244        }
3245    }
3246
3247    /// The swap gate for a HELLO claiming `module_id`.
3248    ///
3249    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
3250    /// presents the candidate nonce, which the reserved gate (holding the
3251    /// incumbent's nonce) would refuse as `reserved_module` before swap
3252    /// admission was ever reached. And it applies to unreserved ids too: for an
3253    /// unreserved id the only thing that ever stopped a second process claiming
3254    /// a live id was the `duplicate_module_id` refusal, which is exactly the
3255    /// refusal a swap lifts for its candidate.
3256    ///
3257    /// The incumbent's own nonce falls through to the ordinary gates, which
3258    /// treat it as they always have (a live incumbent is refused as a
3259    /// duplicate). Anything else while a swap is open is refused, including an
3260    /// absent nonce.
3261    pub(crate) fn swap_hello_admission(
3262        &self,
3263        module_id: &str,
3264        presented: Option<&str>,
3265    ) -> SwapHelloAdmission {
3266        if self.protocol_none(module_id) {
3267            return SwapHelloAdmission::ProtocolNone;
3268        }
3269        if self.module_is_gated(module_id) {
3270            return SwapHelloAdmission::ChannelRequired;
3271        }
3272        let swaps = self
3273            .swaps
3274            .lock()
3275            .unwrap_or_else(|poisoned| poisoned.into_inner());
3276        let Some(swap) = swaps.get(module_id) else {
3277            return SwapHelloAdmission::NotSwapping;
3278        };
3279        let Some(presented) = presented.filter(|nonce| !nonce.is_empty()) else {
3280            return SwapHelloAdmission::Refused;
3281        };
3282        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
3283            return if swap.candidate_admitted {
3284                SwapHelloAdmission::Refused
3285            } else {
3286                SwapHelloAdmission::Candidate
3287            };
3288        }
3289        if swap
3290            .incumbent_nonce
3291            .as_deref()
3292            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
3293        {
3294            return SwapHelloAdmission::NotSwapping;
3295        }
3296        SwapHelloAdmission::Refused
3297    }
3298
3299    fn protocol_none(&self, module: &str) -> bool {
3300        self.get(module)
3301            .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
3302    }
3303
3304    /// Exclude swap closure through the short, synchronous candidate publication.
3305    pub(crate) fn with_swap_candidate_admission<T>(
3306        &self,
3307        module_id: &str,
3308        publish: impl FnOnce() -> (T, bool),
3309    ) -> Option<T> {
3310        let mut swaps = self.swaps.lock().unwrap_or_else(|p| p.into_inner());
3311        let swap = swaps
3312            .get_mut(module_id)
3313            .filter(|swap| !swap.candidate_admitted)?;
3314        let (result, admitted) = publish();
3315        swap.candidate_admitted = admitted;
3316        Some(result)
3317    }
3318
3319    /// Test/support lookup for the current launch nonce of a supervised spawn.
3320    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3321        self.spawn_nonce(module_id)
3322    }
3323
3324    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
3325    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3326        if self.module_is_gated(module_id) {
3327            return None;
3328        }
3329        self.reserved_nonces
3330            .lock()
3331            .unwrap_or_else(|poisoned| poisoned.into_inner())
3332            .get(module_id)
3333            .cloned()
3334            .flatten()
3335    }
3336
3337    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
3338        // Normally already marked before the process was spawned; marking here
3339        // too keeps `configured_ids` a superset of the roster for any caller
3340        // that inserts a module directly.
3341        self.mark_configured(module.module_id());
3342        let mut modules = self
3343            .modules
3344            .lock()
3345            .unwrap_or_else(|poisoned| poisoned.into_inner());
3346        modules.insert(module.module_id().to_string(), module)
3347    }
3348
3349    /// Record that the supervisor has taken on `module_id`. Called before the
3350    /// module's first process is spawned, so that by the time that process can
3351    /// register, [`Self::is_configured`] already answers true.
3352    fn mark_configured(&self, module_id: &str) {
3353        self.update_configured_ids(|ids| {
3354            ids.insert(module_id.to_string());
3355        });
3356    }
3357
3358    fn publish_configured_ids(&self, ids: Option<&HashSet<String>>) {
3359        let owner = self
3360            .gate_runtime
3361            .lock()
3362            .unwrap_or_else(|p| p.into_inner())
3363            .clone()
3364            .and_then(|owner| owner.upgrade());
3365        if let Some(owner) = owner {
3366            owner.observe_supervisor_configured_ids(ids, Arc::downgrade(&self.configured_ids));
3367        }
3368    }
3369
3370    fn update_configured_ids(&self, update: impl FnOnce(&mut HashSet<String>)) {
3371        match self.configured_ids.lock() {
3372            Ok(mut ids) => {
3373                update(&mut ids);
3374                // Publish while this roster mutation is serialized, so an
3375                // older snapshot can never overwrite a later configured id.
3376                self.publish_configured_ids(Some(&ids));
3377            }
3378            Err(poisoned) => {
3379                update(&mut poisoned.into_inner());
3380                self.publish_configured_ids(None);
3381            }
3382        }
3383    }
3384
3385    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
3386    /// before it was ever put on the roster. A module already on the roster
3387    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
3388    fn unmark_configured_unless_rostered(&self, module_id: &str) {
3389        let modules = self
3390            .modules
3391            .lock()
3392            .unwrap_or_else(|poisoned| poisoned.into_inner());
3393        if !modules.contains_key(module_id) {
3394            self.update_configured_ids(|ids| {
3395                ids.remove(module_id);
3396            });
3397        }
3398    }
3399
3400    /// Whether `module_id` is a module this daemon supervises: on the roster,
3401    /// or about to be (its process is being spawned right now).
3402    ///
3403    /// This, not `get(..).is_some()`, is what "the owner is configured" means
3404    /// for scopes: a supervised module's process can register and sync before
3405    /// [`Self::get`] can return it, and in that window it is still a module
3406    /// that will sync, not one that never will.
3407    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
3408        self.configured_ids
3409            .lock()
3410            .unwrap_or_else(|poisoned| poisoned.into_inner())
3411            .contains(module_id)
3412    }
3413
3414    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
3415        let modules = self
3416            .modules
3417            .lock()
3418            .unwrap_or_else(|poisoned| poisoned.into_inner());
3419        modules.get(module_id).cloned()
3420    }
3421
3422    pub(crate) fn record_late_health_answer(
3423        &self,
3424        module_id: &str,
3425        latency_ms: u64,
3426    ) -> Result<bool, SuperviseError> {
3427        let Some(module) = self.get(module_id) else {
3428            return Ok(false);
3429        };
3430        self.host_health.record_late_module(module_id);
3431        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
3432            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
3433            state.health.last_late_answer_latency_ms = Some(latency_ms);
3434            // A late answer is an answer: the module served the probe, just past
3435            // the deadline. Leaving the miss streak in place while logging
3436            // "proves the module is alive" is how a CPU-starved module that
3437            // answers every probe a few seconds late still marches to the
3438            // threshold and gets killed — the exact kill class `NoAnswer` is
3439            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
3440            // is degradation, and degradation reports; it does not restart.
3441            state.health.consecutive_failures = 0;
3442        })?;
3443        Ok(true)
3444    }
3445
3446    pub(crate) fn spawn_host_health_monitor(&self) -> JoinHandle<()> {
3447        self.host_health.spawn_monitor()
3448    }
3449
3450    /// Arm the one-shot marker for the module process that this caller
3451    /// deliberately initiated severance against. Generic connection teardown
3452    /// must not call this:
3453    /// a surviving process would otherwise retain an exemption for a later
3454    /// genuine crash.
3455    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
3456        let Some(module) = self.get(module_id) else {
3457            return Ok(false);
3458        };
3459        let snapshot = lock_snapshot(&module.inner.snapshot)?;
3460        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
3461            return Ok(false);
3462        };
3463        drop(snapshot);
3464        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
3465    }
3466
3467    pub fn list(&self) -> Vec<SupervisedModule> {
3468        let modules = self
3469            .modules
3470            .lock()
3471            .unwrap_or_else(|poisoned| poisoned.into_inner());
3472        let mut modules = modules.values().cloned().collect::<Vec<_>>();
3473        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
3474        modules
3475    }
3476
3477    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
3478        self.activity_images
3479            .lock()
3480            .unwrap_or_else(|p| p.into_inner())
3481            .remove(module_id);
3482        self.spawn_nonces
3483            .lock()
3484            .unwrap_or_else(|poisoned| poisoned.into_inner())
3485            .remove(module_id);
3486        self.close_swap(module_id);
3487        let mut reserved_nonces = self
3488            .reserved_nonces
3489            .lock()
3490            .unwrap_or_else(|poisoned| poisoned.into_inner());
3491        if reserved_nonces.contains_key(module_id) {
3492            // The old nonce must die with the removed process, but the exact-id
3493            // gate remains until an operator explicitly releases it.
3494            reserved_nonces.insert(module_id.to_string(), None);
3495        }
3496        drop(reserved_nonces);
3497        self.reserved_prefix_owners
3498            .lock()
3499            .unwrap_or_else(|poisoned| poisoned.into_inner())
3500            .retain(|_, owner| owner != module_id);
3501        let removed = self
3502            .modules
3503            .lock()
3504            .unwrap_or_else(|poisoned| poisoned.into_inner())
3505            .remove(module_id);
3506        self.update_configured_ids(|ids| {
3507            ids.remove(module_id);
3508        });
3509        self.remove_program_binding_records(module_id);
3510        removed
3511    }
3512
3513    /// Remember a module removed by a non-preview rescan so route.open can
3514    /// distinguish that intentional removal from an unknown id.
3515    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
3516        self.removal_tombstones
3517            .lock()
3518            .unwrap_or_else(|poisoned| poisoned.into_inner())
3519            .insert(module_id.to_string(), unix_ms_now());
3520    }
3521
3522    /// Return how long ago a rescan removed this module in milliseconds.
3523    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
3524        self.removal_tombstones
3525            .lock()
3526            .unwrap_or_else(|poisoned| poisoned.into_inner())
3527            .get(module_id)
3528            .copied()
3529            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
3530    }
3531
3532    /// Retire a reserved-id gate only after its module has left supervision.
3533    ///
3534    /// A retained gate has no live nonce (`None`), so releasing any other entry
3535    /// would weaken a currently configured or otherwise active reservation.
3536    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
3537        if self.get(module_id).is_some() {
3538            return false;
3539        }
3540        let mut reserved_nonces = self
3541            .reserved_nonces
3542            .lock()
3543            .unwrap_or_else(|poisoned| poisoned.into_inner());
3544        if !matches!(reserved_nonces.get(module_id), Some(None)) {
3545            return false;
3546        }
3547        reserved_nonces.remove(module_id);
3548        self.implicit_protocol_none
3549            .lock()
3550            .unwrap_or_else(|p| p.into_inner())
3551            .remove(module_id);
3552        true
3553    }
3554
3555    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
3556        Arc::clone(&self.operation_lock)
3557    }
3558}
3559
3560/// Process supervisor for subc-owned singleton modules.
3561#[derive(Debug, Clone)]
3562pub struct Supervisor {
3563    registry: Arc<Registry>,
3564    config_delivery: Option<config_delivery::ConfigDelivery>,
3565    restart_policy: RestartPolicy,
3566    drain_timeout: Duration,
3567    connection_file_path: Option<PathBuf>,
3568    capture_logs_dir: Option<PathBuf>,
3569    forwarding: Option<Arc<ForwardingTable>>,
3570    process_liveness: Arc<SupervisorProcessLiveness>,
3571    supervisor_handle: Option<SupervisorHandle>,
3572    health: HealthConfig,
3573    host_health: HostHealth,
3574    daemon_start_clock: crate::clock::StartClock,
3575    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
3576    spawn_events: SpawnEventFeed,
3577    provenance_probe: ExecutableIdentityProbe,
3578    /// Every process spawned through this supervisor (and its clones) and not
3579    /// yet reaped, so daemon shutdown can end them.
3580    child_roster: ChildRoster,
3581    #[cfg(target_os = "linux")]
3582    cgroup_placement: Option<subc_cgroup::Placement>,
3583    #[cfg(test)]
3584    test_after_first_spawn: AfterFirstSpawnHook,
3585}
3586
3587/// A roster entry whose monitor and first process have not started yet.
3588/// Keeping the command receiver here lets discovery serve status immediately
3589/// while start/stop requests wait until bootstrap has published the connection file.
3590pub(crate) struct RegisteredConfiguredModule {
3591    spec: ModuleSpec,
3592    runtime: SupervisorRuntimeConfig,
3593    module: SupervisedModule,
3594    commands: mpsc::Receiver<SupervisorCommand>,
3595}
3596
3597/// Test-only hook run on the path that takes on a new module, right after its
3598/// first `spawn_child` returns (the process exists and could already be
3599/// registering) and before that process is handed to the module's supervise
3600/// loop and put on the roster. Lets a test observe what a fast child would see
3601/// in that window without racing a real one.
3602#[cfg(test)]
3603#[derive(Clone, Default)]
3604struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
3605
3606#[cfg(test)]
3607type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
3608
3609#[cfg(test)]
3610impl fmt::Debug for AfterFirstSpawnHook {
3611    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3612        f.write_str("AfterFirstSpawnHook")
3613    }
3614}
3615
3616#[cfg(test)]
3617impl AfterFirstSpawnHook {
3618    fn run(&self, module_id: &str) {
3619        if let Some(hook) = &self.0 {
3620            hook(module_id);
3621        }
3622    }
3623}
3624
3625impl Supervisor {
3626    #[cfg(all(test, target_os = "linux"))]
3627    pub(crate) fn set_admission_hook_for_test(
3628        &self,
3629        hook: impl Fn(&str, &'static str, Option<u32>) -> bool + Send + Sync + 'static,
3630    ) {
3631        self.child_roster.set_admission_hook_for_test(hook);
3632    }
3633
3634    #[cfg(test)]
3635    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
3636        let supervisor = Self::new(registry, policy);
3637        #[cfg(target_os = "macos")]
3638        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
3639        supervisor
3640    }
3641    /// Verify the trampoline once when configured. Missing private OS support
3642    /// refuses every macOS launch by name but does not stop the daemon's control
3643    /// server. Embedders must explicitly provide a binary with the subc-os hidden
3644    /// entry point; the library must not exec an arbitrary hosting program.
3645    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
3646        let path = path.into();
3647        #[cfg(target_os = "macos")]
3648        {
3649            let result = probe_privacy_trampoline(&path).map(|()| path);
3650            if let Err(cause) = &result {
3651                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
3652            }
3653            self.child_roster.set_privacy_trampoline(result);
3654        }
3655        #[cfg(not(target_os = "macos"))]
3656        let _ = path;
3657        self
3658    }
3659    /// The flag [`Self::begin_daemon_shutdown`] sets, for the control handler's
3660    /// capability-requirement log to read.
3661    pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
3662        self.child_roster.shutdown_flag()
3663    }
3664
3665    /// The first step of an announced daemon shutdown, before the notice and
3666    /// before any connection is closed.
3667    ///
3668    /// Sets the daemon-shutdown flag first: from here on no module is
3669    /// respawned (crash restart, operator restart, or swap), and every child
3670    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
3671    /// the module exits on the EOF this shutdown gives it or is signalled by a
3672    /// service manager that kills the whole cgroup. Then writes the journal's
3673    /// shutdown marker, which records the instant and closes this daemon
3674    /// incarnation's stretch of the journal.
3675    #[cfg(unix)]
3676    pub(crate) fn begin_daemon_shutdown(&self) {
3677        self.child_roster.close();
3678        if let Some(journal) = &self.terminal_journal {
3679            journal.stamp_shutdown();
3680        }
3681    }
3682
3683    /// Announce a cut while established connections can still carry replies.
3684    /// These budgets bound the attempt to deliver notices and the wait for
3685    /// routes to drain; neither guarantees delivery or child completion, and
3686    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
3687    #[cfg(unix)]
3688    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
3689        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
3690        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
3691        let Some(forwarding) = &self.forwarding else {
3692            return Ok(());
3693        };
3694        let notice_started = Instant::now();
3695        let modules = forwarding
3696            .begin_daemon_drain()
3697            .map_err(SuperviseError::Forwarding)?;
3698        let deadline_ms =
3699            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
3700        let shutdown_reason = self
3701            .supervisor_handle
3702            .as_ref()
3703            .map(SupervisorHandle::take_daemon_shutdown_reason)
3704            .unwrap_or(RouteCloseReason::Disable);
3705        let mut modules_told = 0;
3706        let mut notices = tokio::task::JoinSet::new();
3707        let mut drains = Vec::new();
3708        for connection in forwarding
3709            .module_connections()
3710            .map_err(SuperviseError::Forwarding)?
3711        {
3712            let target = ModuleDrainTarget {
3713                endpoint: connection.endpoint,
3714                sink: connection.sink,
3715                negotiated_ver: connection.negotiated_ver,
3716                abandoned_bindings: Vec::new(),
3717                excluded_subscriptions: 0,
3718            };
3719            notices.spawn(async move {
3720                (
3721                    true,
3722                    send_module_draining(
3723                        &connection.module_id,
3724                        shutdown_reason,
3725                        deadline_ms,
3726                        &target,
3727                        true,
3728                    )
3729                    .await,
3730                )
3731            });
3732        }
3733        for (module_id, endpoint) in modules {
3734            let Some(target) = forwarding
3735                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
3736                .map_err(SuperviseError::Forwarding)?
3737            else {
3738                continue;
3739            };
3740            let routes = forwarding
3741                .endpoint_routes(target.endpoint)
3742                .map_err(SuperviseError::Forwarding)?;
3743            // Only an announced restart promises that modules are coming back.
3744            // Client route notices retain their existing reconnect semantics.
3745            let mut recipients = Vec::new();
3746            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
3747            for route in routes {
3748                let client = route.goodbye_target;
3749                if let Some((_, channels)) = clients
3750                    .iter_mut()
3751                    .find(|(existing, _)| existing.connection_id == client.connection_id)
3752                {
3753                    channels.push(client.channel);
3754                } else {
3755                    let channel = client.channel;
3756                    clients.push((client, vec![channel]));
3757                }
3758            }
3759            for (client, mut channels) in clients {
3760                channels.sort_unstable();
3761                channels.dedup();
3762                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
3763                    module_id: module_id.clone(),
3764                    channels,
3765                    reason: RouteCloseReason::Restart,
3766                })
3767                .expect("route closing serializes");
3768                recipients.push((client.sink, client.negotiated_ver, closing));
3769            }
3770            for (sink, version, body) in recipients {
3771                notices.spawn(async move {
3772                    let frame = Frame::build_with_version(
3773                        version,
3774                        FrameType::Push,
3775                        control_flags(),
3776                        0,
3777                        0,
3778                        0,
3779                        body,
3780                    )
3781                    .expect("bounded lifecycle notice frame builds");
3782                    (false, sink.send_flushed(frame).await.is_ok())
3783                });
3784            }
3785            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3786            drains.push((module_id, target.endpoint, gauges));
3787        }
3788        // A quiet forwarding table is not proof that queued notices reached the
3789        // socket. Wait for writer flush acknowledgements before testing quiescence.
3790        let notice_deadline = Instant::now() + NOTICE_BUDGET;
3791        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3792            match result {
3793                Ok((true, true)) => modules_told += 1,
3794                Ok((false, true)) => {}
3795                result => warn!(?result, "daemon shutdown notice delivery failed"),
3796            }
3797        }
3798        notices.abort_all();
3799        info!(
3800            ?shutdown_reason,
3801            modules_told, "daemon shutdown module notices delivered"
3802        );
3803        let notice_elapsed = notice_started.elapsed();
3804        let drain_started = Instant::now();
3805        let deadline = drain_started + DRAIN_BUDGET;
3806        let mut waits = tokio::task::JoinSet::new();
3807        for (module_id, endpoint, gauges) in drains {
3808            let forwarding = Arc::clone(forwarding);
3809            let mut runtime = self.runtime_config();
3810            runtime.health.cadence = Duration::from_millis(100);
3811            waits.spawn(async move {
3812                let wait_result = wait_for_forwarding_quiescence(
3813                    &forwarding,
3814                    &module_id,
3815                    &runtime,
3816                    endpoint,
3817                    deadline,
3818                    &gauges,
3819                    DrainScope::Endpoint(endpoint),
3820                    &mut DrainGaugeTrace::new(&gauges),
3821                )
3822                .await;
3823                // Store the measurement before the task returns: quiescence
3824                // may be reached just as the deadline stops the join loop from
3825                // collecting results. Unfinished or failed waits report false.
3826                forwarding
3827                    .record_daemon_drain_result(
3828                        endpoint,
3829                        drained_after_quiescence_wait(&wait_result),
3830                    )
3831                    .map_err(SuperviseError::Forwarding)?;
3832                wait_result
3833            });
3834        }
3835        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3836            if !matches!(result, Ok(Ok(true))) {
3837                warn!(?result, "daemon shutdown drain did not reach quiescence");
3838            }
3839        }
3840        // Measure only the forwarding drain, not child teardown or the caller's
3841        // signal delivery and process-reaping delays.
3842        info!(
3843            notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3844            drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3845            "daemon shutdown forwarding drain complete"
3846        );
3847        Ok(())
3848    }
3849
3850    /// The last step of a graceful daemon shutdown, after the notice and the
3851    /// drain: send every registered module a module GOODBYE, the same planned
3852    /// stop signal `ck module stop` gives, then close every connection so each
3853    /// subc module sees EOF and starts its own teardown, then end every
3854    /// supervised child that has not exited
3855    /// by its own deadline (its drain budget, capped). Modules lead their own
3856    /// process groups, so a
3857    /// service manager's group kill no longer reaches them; without this a
3858    /// child that does not stop on EOF (every `protocol: "none"` child, which
3859    /// has no connection) would outlive the daemon. Every wait is bounded (see
3860    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
3861    #[cfg(unix)]
3862    pub(crate) async fn end_children_for_daemon_shutdown(
3863        &self,
3864        already_escalated: bool,
3865        escalate: impl std::future::Future<Output = ()>,
3866    ) {
3867        tokio::pin!(escalate);
3868        let mut escalated = already_escalated;
3869        if let Some(forwarding) = &self.forwarding {
3870            let reason = CloseReason::new(
3871                "daemon_shutdown",
3872                "the daemon is exiting after its shutdown notice and drain",
3873            );
3874            if escalated {
3875                // The operator asked to stop waiting: queue the GOODBYEs but
3876                // do not wait for them to be written.
3877                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3878            } else {
3879                tokio::select! {
3880                    biased;
3881                    _ = escalate.as_mut() => {
3882                        info!("second SIGTERM: abandoning module GOODBYE delivery");
3883                        escalated = true;
3884                    }
3885                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3886                }
3887            }
3888            let closed = forwarding.close_all_connections(&reason);
3889            debug!(closed, "closed established connections for daemon shutdown");
3890        }
3891        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
3892        // already completed and must not be polled again; the child shutdown
3893        // wait is told it is escalated and gets a future that never fires.
3894        let escalated_here = escalated && !already_escalated;
3895        let remaining_escalate = async move {
3896            if escalated_here {
3897                std::future::pending::<()>().await;
3898            } else {
3899                escalate.await;
3900            }
3901        };
3902        crate::child_roster::end_children_for_daemon_shutdown(
3903            &self.child_roster,
3904            escalated,
3905            remaining_escalate,
3906        )
3907        .await;
3908    }
3909
3910    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3911        Self {
3912            registry,
3913            config_delivery: None,
3914            restart_policy,
3915            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3916            connection_file_path: None,
3917            capture_logs_dir: None,
3918            forwarding: None,
3919            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3920            supervisor_handle: None,
3921            health: HealthConfig::default(),
3922            host_health: HostHealth::default(),
3923            daemon_start_clock: crate::clock::StartClock::capture(),
3924            terminal_journal: None,
3925            spawn_events: SpawnEventFeed::default(),
3926            provenance_probe: ExecutableIdentityProbe::default(),
3927            child_roster: ChildRoster::default(),
3928            #[cfg(target_os = "linux")]
3929            cgroup_placement: None,
3930            #[cfg(test)]
3931            test_after_first_spawn: AfterFirstSpawnHook::default(),
3932        }
3933    }
3934
3935    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3936        self.drain_timeout = drain_timeout;
3937        self
3938    }
3939
3940    /// The drain budget a module gets when its config entry sets none.
3941    pub(crate) fn default_drain_timeout(&self) -> Duration {
3942        self.drain_timeout
3943    }
3944
3945    pub fn with_process_liveness(
3946        mut self,
3947        process_liveness: Arc<SupervisorProcessLiveness>,
3948    ) -> Self {
3949        self.process_liveness = process_liveness;
3950        self
3951    }
3952
3953    /// Connect configuration delivery to observed process lifecycle events.
3954    pub fn with_config_runtime(
3955        mut self,
3956        owner: Arc<crate::config_gate::runtime::RuntimeOwner>,
3957    ) -> Self {
3958        if let Some(handle) = &self.supervisor_handle {
3959            handle.install_binding_retirement(&owner);
3960        }
3961        self.config_delivery = Some(config_delivery::ConfigDelivery(owner));
3962        self
3963    }
3964
3965    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3966        self.connection_file_path = Some(connection_file_path.into());
3967        self
3968    }
3969
3970    /// Enables daemon-owned capture files for supervised stdout and stderr.
3971    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3972        self.capture_logs_dir = Some(logs_dir.into());
3973        self
3974    }
3975
3976    /// Names this daemon lifetime in spawn events, independently of whether a
3977    /// terminal journal is configured.
3978    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3979        // A millisecond start stamp can repeat after clock rollback or a rapid
3980        // restart. Use the connection file's random daemon_id instead: it already
3981        // identifies this daemon lifetime independently of the wall clock.
3982        self.spawn_events.configure_incarnation(daemon_incarnation);
3983        self
3984    }
3985
3986    /// Enables best-effort history shared by every supervised module. Without
3987    /// it, terminal history is kept only in each module's in-memory ring.
3988    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3989        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3990        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3991            path,
3992            daemon_incarnation,
3993        )));
3994        this
3995    }
3996
3997    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3998        self.child_roster.counters = forwarding.counters();
3999        self.forwarding = Some(forwarding);
4000        self
4001    }
4002
4003    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
4004        if let Some(delivery) = &self.config_delivery {
4005            supervisor_handle.install_binding_retirement(&delivery.0);
4006        }
4007        self.host_health = supervisor_handle.host_health.clone();
4008        self.spawn_events = supervisor_handle.spawn_events.clone();
4009        self.supervisor_handle = Some(supervisor_handle);
4010        self
4011    }
4012
4013    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
4014        self.health = health;
4015        self
4016    }
4017
4018    /// Keeps a record of every live child at `path`, rewritten on each spawn and
4019    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
4020    /// record is kept.
4021    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
4022        self.child_roster.record_to(path.into());
4023        self
4024    }
4025
4026    #[cfg(target_os = "linux")]
4027    pub fn with_cgroup_placement(
4028        mut self,
4029        cgroup_placement: Option<subc_cgroup::Placement>,
4030    ) -> Self {
4031        self.child_roster
4032            .set_cgroup_placement(cgroup_placement.clone());
4033        self.cgroup_placement = cgroup_placement;
4034        self
4035    }
4036
4037    /// Spawn `spec.program` and start monitoring it.
4038    ///
4039    /// The child is expected to parse `--subc <connection-file-path>`, read the
4040    /// TCP+key connection file, authenticate to the already-running listener, and
4041    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
4042    ///
4043    /// This synchronous embedder API blocks on launch admission and store I/O.
4044    /// Do not call it on an async runtime thread; use [`Self::spawn_async`].
4045    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4046        validate_spec(&spec)?;
4047        self.establish_identity(&spec);
4048
4049        let runtime = self.runtime_config();
4050        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4051        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4052        let registration_watch = watch_registration(&self.registry, &spec);
4053        let spawned = spawn_child(
4054            &spec,
4055            runtime.connection_file_path.as_deref(),
4056            self.supervisor_handle.as_ref(),
4057            &runtime.stderr_ring,
4058            runtime.capture_logs_dir.as_deref(),
4059            &runtime.child_roster,
4060            runtime.spawn_delivery(),
4061            #[cfg(target_os = "linux")]
4062            runtime.cgroup_placement.as_ref(),
4063            runtime.register_timeout,
4064        );
4065        #[cfg(test)]
4066        self.test_after_first_spawn.run(&spec.module_id);
4067        let mut child = match spawned {
4068            Ok(child) => child,
4069            Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4070                defer_config_spawn(
4071                    &spec,
4072                    &runtime,
4073                    &snapshot,
4074                    RespawnKind::Spawn,
4075                    &runtime.failure_context(runtime.recovery_is_daemon),
4076                )?;
4077                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4078            }
4079            Err(SuperviseError::ConfigHeld { .. }) => {
4080                hold_config_spawn(
4081                    &spec,
4082                    &runtime,
4083                    &snapshot,
4084                    &runtime.failure_context(runtime.recovery_is_daemon),
4085                )?;
4086                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4087            }
4088            Err(err) => {
4089                // Unlike the configured paths, a failed `spawn` leaves nothing
4090                // on the roster, so the module must not stay marked configured.
4091                self.abandon_unrostered(&spec.module_id);
4092                return Err(err);
4093            }
4094        };
4095        arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4096        self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4097        if let Some(spawn) = activity_spawn {
4098            spawn.complete(&snapshot);
4099        }
4100
4101        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4102    }
4103
4104    /// Spawn without running configuration integrity or keychain checks on the
4105    /// runtime thread. Cancellation during preparation cannot create a child.
4106    pub async fn spawn_async(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4107        validate_spec(&spec)?;
4108        self.establish_identity(&spec);
4109        let mut identity = AsyncInitialIdentity {
4110            supervisor: self,
4111            module: spec.module_id.clone(),
4112            committed: false,
4113        };
4114        let runtime = self.runtime_config();
4115        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4116        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4117        let registration_watch = watch_registration(&self.registry, &spec);
4118        let spawned = spawn_child_async(&spec, &runtime).await;
4119        #[cfg(test)]
4120        self.test_after_first_spawn.run(&spec.module_id);
4121        let mut child = match spawned {
4122            Ok(child) => child,
4123            Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4124                defer_config_spawn(
4125                    &spec,
4126                    &runtime,
4127                    &snapshot,
4128                    RespawnKind::Spawn,
4129                    &runtime.failure_context(runtime.recovery_is_daemon),
4130                )?;
4131                identity.committed = true;
4132                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4133            }
4134            Err(SuperviseError::ConfigHeld { .. }) => {
4135                hold_config_spawn(
4136                    &spec,
4137                    &runtime,
4138                    &snapshot,
4139                    &runtime.failure_context(runtime.recovery_is_daemon),
4140                )?;
4141                identity.committed = true;
4142                return Ok(self.supervised_module(spec, runtime, snapshot, None));
4143            }
4144            Err(err) => {
4145                self.abandon_unrostered(&spec.module_id);
4146                return Err(err);
4147            }
4148        };
4149        arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4150        self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4151        if let Some(spawn) = activity_spawn {
4152            spawn.complete(&snapshot);
4153        }
4154        identity.committed = true;
4155        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4156    }
4157
4158    /// Make `spec`'s module count as configured, with its identity gates
4159    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
4160    /// exists.
4161    ///
4162    /// Every path that takes on a new module calls this before `spawn_child`.
4163    /// The order is the point: the child can connect, register, sync its
4164    /// scopes and ask about them as soon as it is spawned, and the module is
4165    /// only put on the roster after `spawn_child` returns. Were the mark set
4166    /// with the roster entry, a fast child would see its own owner reported
4167    /// as not configured, and a scoped `route.open` in that window would be
4168    /// refused as terminal `scope_not_live` ("will never sync") instead of
4169    /// retryable `scope_not_synced`.
4170    fn establish_identity(&self, spec: &ModuleSpec) {
4171        if let Some(supervisor_handle) = &self.supervisor_handle {
4172            supervisor_handle.apply_identity_configuration(spec);
4173            supervisor_handle.mark_configured(&spec.module_id);
4174        }
4175    }
4176
4177    /// Take back [`Self::establish_identity`]'s configured mark when the
4178    /// module will not be put on the roster after all.
4179    fn abandon_unrostered(&self, module_id: &str) {
4180        if let Some(supervisor_handle) = &self.supervisor_handle {
4181            supervisor_handle.unmark_configured_unless_rostered(module_id);
4182        }
4183    }
4184
4185    /// Record a freshly spawned first process as running. On failure the
4186    /// module never reaches the roster, so its configured mark is taken back.
4187    fn mark_first_process_running(
4188        &self,
4189        spec: &ModuleSpec,
4190        runtime: &SupervisorRuntimeConfig,
4191        snapshot: &SharedSnapshot,
4192        child: &mut SupervisedChild,
4193    ) -> Result<(), SuperviseError> {
4194        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
4195            containment::terminate_blocking(&runtime.child_roster.containment);
4196            self.abandon_unrostered(&spec.module_id);
4197            return Err(err);
4198        }
4199        self.process_liveness
4200            .track(spec.module_id.clone(), Arc::clone(snapshot));
4201        Ok(())
4202    }
4203
4204    /// Start supervising a module declared in daemon configuration.
4205    ///
4206    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
4207    /// failures in the supervisor handle so operator-facing `supervisor.list`
4208    /// reflects every configured module while daemon startup continues.
4209    pub fn supervise_configured(
4210        &self,
4211        spec: ModuleSpec,
4212        enabled: bool,
4213    ) -> Result<SupervisedModule, SuperviseError> {
4214        self.supervise_configured_requested(
4215            spec,
4216            enabled,
4217            self.health.clone(),
4218            None,
4219            self.restart_policy,
4220            DEFAULT_REGISTER_TIMEOUT,
4221            None,
4222        )
4223    }
4224
4225    /// Supervise a configured module with its own health, drain, and crash
4226    /// budget. The restart policy is per-module because the config file is:
4227    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
4228    /// module that is expensive to restart should not be forced onto the same
4229    /// budget as one that is cheap.
4230    pub fn supervise_configured_with_health(
4231        &self,
4232        spec: ModuleSpec,
4233        enabled: bool,
4234        health: HealthConfig,
4235        drain_timeout_ms: Option<u64>,
4236        restart_policy: RestartPolicy,
4237    ) -> Result<SupervisedModule, SuperviseError> {
4238        self.supervise_configured_requested(
4239            spec,
4240            enabled,
4241            health,
4242            drain_timeout_ms,
4243            restart_policy,
4244            DEFAULT_REGISTER_TIMEOUT,
4245            None,
4246        )
4247    }
4248
4249    #[allow(clippy::too_many_arguments)]
4250    pub(crate) fn supervise_configured_requested(
4251        &self,
4252        spec: ModuleSpec,
4253        enabled: bool,
4254        health: HealthConfig,
4255        drain_timeout_ms: Option<u64>,
4256        restart_policy: RestartPolicy,
4257        register_timeout: Duration,
4258        requester: Option<Principal>,
4259    ) -> Result<SupervisedModule, SuperviseError> {
4260        let registered = self.register_configured_with_health(
4261            spec,
4262            enabled,
4263            health,
4264            drain_timeout_ms,
4265            restart_policy,
4266            register_timeout,
4267        )?;
4268        let daemon_initiated = requester.is_none();
4269        self.start_registered_configured(registered, requester, daemon_initiated)
4270    }
4271
4272    #[allow(clippy::too_many_arguments)]
4273    pub(crate) async fn supervise_configured_requested_async(
4274        &self,
4275        spec: ModuleSpec,
4276        enabled: bool,
4277        health: HealthConfig,
4278        drain_timeout_ms: Option<u64>,
4279        restart_policy: RestartPolicy,
4280        register_timeout: Duration,
4281        requester: Option<Principal>,
4282    ) -> Result<SupervisedModule, SuperviseError> {
4283        let registered = self.register_configured_with_health(
4284            spec,
4285            enabled,
4286            health,
4287            drain_timeout_ms,
4288            restart_policy,
4289            register_timeout,
4290        )?;
4291        let daemon_initiated = requester.is_none();
4292        self.start_registered_configured_async(registered, requester, daemon_initiated)
4293            .await
4294    }
4295
4296    /// Record a configured module as starting or disabled, without launching a
4297    /// process or running its monitor. Bootstrap registers the entire roster
4298    /// before publishing discovery, then activates it once children can read
4299    /// the connection file passed in their `--subc` argument.
4300    pub(crate) fn register_configured_with_health(
4301        &self,
4302        spec: ModuleSpec,
4303        enabled: bool,
4304        health: HealthConfig,
4305        drain_timeout_ms: Option<u64>,
4306        restart_policy: RestartPolicy,
4307        register_timeout: Duration,
4308    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
4309        validate_spec(&spec)?;
4310        self.establish_identity(&spec);
4311
4312        let mut runtime = self.runtime_config();
4313        runtime.health = health;
4314        runtime.restart_policy = restart_policy;
4315        runtime.register_timeout = register_timeout;
4316        if let Some(ms) = drain_timeout_ms {
4317            runtime.drain_timeout = Duration::from_millis(ms);
4318            *runtime
4319                .effective_drain_timeout
4320                .lock()
4321                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
4322        }
4323        let snapshot = if enabled {
4324            SupervisorSnapshot::starting()
4325        } else {
4326            SupervisorSnapshot::disabled()
4327        };
4328        let snapshot = {
4329            let mut snapshot = snapshot;
4330            if enabled {
4331                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
4332                {
4333                    snapshot.waiting_for_previous_pid = Some(pid);
4334                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
4335                }
4336            }
4337            snapshot
4338        };
4339        let (module, commands) =
4340            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
4341        Ok(RegisteredConfiguredModule {
4342            spec,
4343            runtime,
4344            module,
4345            commands,
4346        })
4347    }
4348
4349    /// Activate a previously registered module. Queued control requests are
4350    /// served by the monitor after the initial configured spawn has settled.
4351    pub(crate) fn start_registered_configured(
4352        &self,
4353        registered: RegisteredConfiguredModule,
4354        requester: Option<Principal>,
4355        daemon_initiated: bool,
4356    ) -> Result<SupervisedModule, SuperviseError> {
4357        let RegisteredConfiguredModule {
4358            spec,
4359            mut runtime,
4360            module,
4361            commands,
4362        } = registered;
4363        runtime.start_requester = requester;
4364        runtime.recovery_is_daemon = daemon_initiated;
4365        let snapshot = &module.inner.snapshot;
4366        let should_spawn = {
4367            let state = lock_snapshot(snapshot)?;
4368            state.enabled && state.waiting_for_previous_pid.is_none()
4369        };
4370        let child = if should_spawn {
4371            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4372            let registration_watch = watch_registration(&self.registry, &spec);
4373            let spawned = spawn_child(
4374                &spec,
4375                runtime.connection_file_path.as_deref(),
4376                self.supervisor_handle.as_ref(),
4377                &runtime.stderr_ring,
4378                runtime.capture_logs_dir.as_deref(),
4379                &runtime.child_roster,
4380                runtime.spawn_delivery(),
4381                #[cfg(target_os = "linux")]
4382                runtime.cgroup_placement.as_ref(),
4383                runtime.register_timeout,
4384            );
4385            #[cfg(test)]
4386            self.test_after_first_spawn.run(&spec.module_id);
4387            match spawned {
4388                Ok(mut child) => {
4389                    arm_registration_deadline(
4390                        &mut child,
4391                        runtime.register_timeout,
4392                        registration_watch,
4393                    );
4394                    self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4395                    if let Some(spawn) = activity_spawn {
4396                        spawn.complete(snapshot);
4397                    }
4398                    Some(child)
4399                }
4400                Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4401                    defer_config_spawn(
4402                        &spec,
4403                        &runtime,
4404                        snapshot,
4405                        RespawnKind::Spawn,
4406                        &runtime.failure_context(daemon_initiated),
4407                    )?;
4408                    None
4409                }
4410                Err(mut err) => {
4411                    if matches!(err, SuperviseError::ConfigHeld { .. }) {
4412                        match hold_config_spawn(
4413                            &spec,
4414                            &runtime,
4415                            snapshot,
4416                            &runtime.failure_context(daemon_initiated),
4417                        ) {
4418                            Ok(()) => {
4419                                self.start_monitor(spec, runtime, &module, None, commands);
4420                                return Ok(module);
4421                            }
4422                            Err(parking_error) => err = parking_error,
4423                        }
4424                    }
4425                    if runtime.health.critical {
4426                        error!(
4427                            module_id = %spec.module_id,
4428                            program = %spec.program.display(),
4429                            error = %err,
4430                            "critical configured module failed to spawn; marking failed and alerting"
4431                        );
4432                    } else {
4433                        error!(
4434                            module_id = %spec.module_id,
4435                            program = %spec.program.display(),
4436                            error = %err,
4437                            "configured module failed to spawn; marking failed and continuing"
4438                        );
4439                    }
4440                    fail_snapshot(
4441                        snapshot,
4442                        Some(&spec.module_id),
4443                        None,
4444                        &runtime.failure_context(daemon_initiated),
4445                    );
4446                    None
4447                }
4448            }
4449        } else {
4450            None
4451        };
4452        self.start_monitor(spec, runtime, &module, child, commands);
4453        Ok(module)
4454    }
4455
4456    pub(crate) async fn start_registered_configured_async(
4457        &self,
4458        registered: RegisteredConfiguredModule,
4459        requester: Option<Principal>,
4460        daemon_initiated: bool,
4461    ) -> Result<SupervisedModule, SuperviseError> {
4462        self.start_registered_with_failure_policy(registered, requester, daemon_initiated, None)
4463            .await
4464    }
4465
4466    pub(crate) async fn start_registered_approved_async(
4467        &self,
4468        registered: RegisteredConfiguredModule,
4469        requester: Principal,
4470        generation: crate::config_gate::runtime::ApprovalIdentity,
4471    ) -> Result<SupervisedModule, SuperviseError> {
4472        self.start_registered_with_failure_policy(
4473            registered,
4474            Some(requester),
4475            false,
4476            Some(generation),
4477        )
4478        .await
4479    }
4480
4481    async fn start_registered_with_failure_policy(
4482        &self,
4483        registered: RegisteredConfiguredModule,
4484        requester: Option<Principal>,
4485        daemon_initiated: bool,
4486        approved_generation: Option<crate::config_gate::runtime::ApprovalIdentity>,
4487    ) -> Result<SupervisedModule, SuperviseError> {
4488        let RegisteredConfiguredModule {
4489            spec,
4490            mut runtime,
4491            module,
4492            commands,
4493        } = registered;
4494        runtime.start_requester = requester;
4495        runtime.recovery_is_daemon = daemon_initiated;
4496        let snapshot = &module.inner.snapshot;
4497        let mut startup_error = None;
4498        let failure = FailureContext {
4499            approval: approved_generation.clone(),
4500            ..runtime.failure_context(daemon_initiated)
4501        };
4502        let should_spawn = {
4503            let state = lock_snapshot(snapshot)?;
4504            state.enabled && state.waiting_for_previous_pid.is_none()
4505        };
4506        let child = if should_spawn {
4507            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4508            let registration_watch = watch_registration(&self.registry, &spec);
4509            let spawned = spawn_child_async(&spec, &runtime).await;
4510            #[cfg(test)]
4511            self.test_after_first_spawn.run(&spec.module_id);
4512            match spawned {
4513                Ok(mut child) => {
4514                    arm_registration_deadline(
4515                        &mut child,
4516                        runtime.register_timeout,
4517                        registration_watch,
4518                    );
4519                    self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4520                    if let Some(spawn) = activity_spawn {
4521                        spawn.complete(snapshot);
4522                    }
4523                    Some(child)
4524                }
4525                Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4526                    defer_config_spawn(&spec, &runtime, snapshot, RespawnKind::Spawn, &failure)?;
4527                    None
4528                }
4529                Err(mut err) => {
4530                    if matches!(err, SuperviseError::ConfigHeld { .. }) {
4531                        match hold_config_spawn(&spec, &runtime, snapshot, &failure) {
4532                            Ok(()) => {
4533                                self.start_monitor(spec, runtime, &module, None, commands);
4534                                return Ok(module);
4535                            }
4536                            Err(parking_error) => err = parking_error,
4537                        }
4538                    }
4539                    if runtime.health.critical {
4540                        error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4541                            "critical configured module failed to spawn; marking failed and alerting");
4542                    } else {
4543                        error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4544                            "configured module failed to spawn; marking failed and continuing");
4545                    }
4546                    fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
4547                    if let Some(generation) = approved_generation {
4548                        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4549                            state.approved_start_failure = Some((generation, err.to_string()));
4550                        });
4551                        startup_error = Some(err);
4552                    }
4553                    None
4554                }
4555            }
4556        } else {
4557            None
4558        };
4559        self.start_monitor(spec, runtime, &module, child, commands);
4560        // Applying an operator-approved configuration must report a spawn error
4561        // without removing the failed module or its monitor. Bootstrap instead
4562        // tolerates failed starts so other configured modules can still run.
4563        startup_error.map_or(Ok(module), Err)
4564    }
4565
4566    fn runtime_config(&self) -> SupervisorRuntimeConfig {
4567        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
4568        SupervisorRuntimeConfig {
4569            registry: self.registry.clone(),
4570            config_delivery: self.config_delivery.clone(),
4571            recovery_is_daemon: true,
4572            start_requester: None,
4573            pending_activity_restart: false,
4574            provenance_probe: self.provenance_probe.clone(),
4575            scheduled_respawn: Arc::default(),
4576            deferred_reload_reply: Arc::default(),
4577            restart_policy: self.restart_policy,
4578            register_timeout: DEFAULT_REGISTER_TIMEOUT,
4579            drain_timeout: self.drain_timeout,
4580            // Shared with this module's roster copy: daemon shutdown waits on
4581            // each child for the module's own drain budget, as resolved now.
4582            child_roster: self
4583                .child_roster
4584                .for_module(Arc::clone(&effective_drain_timeout)),
4585            effective_drain_timeout,
4586            default_drain_timeout: self.drain_timeout,
4587            health: self.health.clone(),
4588            host_health: self.host_health.clone(),
4589            connection_file_path: self.connection_file_path.clone(),
4590            capture_logs_dir: self.capture_logs_dir.clone(),
4591            forwarding: self.forwarding.clone(),
4592            supervisor_handle: self.supervisor_handle.clone(),
4593            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
4594            terminal_ring: Arc::new(Mutex::new(
4595                TerminalRing::new(
4596                    TerminalRingConfig::default(),
4597                    self.daemon_start_clock.started_at_ms(),
4598                )
4599                .with_start_clock(self.daemon_start_clock)
4600                .with_journal(self.terminal_journal.clone())
4601                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
4602            )),
4603            spawn_events: self.spawn_events.clone(),
4604            #[cfg(target_os = "linux")]
4605            cgroup_placement: self.cgroup_placement.clone(),
4606            #[cfg(test)]
4607            test_seed_stale_facts_before_enable_spawn: false,
4608            #[cfg(test)]
4609            test_reload_exit_record_gate: None,
4610        }
4611    }
4612
4613    fn supervised_module(
4614        &self,
4615        spec: ModuleSpec,
4616        runtime: SupervisorRuntimeConfig,
4617        snapshot: SharedSnapshot,
4618        child: Option<SupervisedChild>,
4619    ) -> SupervisedModule {
4620        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
4621        self.start_monitor(spec, runtime, &module, child, commands);
4622        module
4623    }
4624
4625    fn register_module(
4626        &self,
4627        spec: &ModuleSpec,
4628        runtime: &SupervisorRuntimeConfig,
4629        snapshot: SharedSnapshot,
4630    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
4631        runtime.host_health.track(spec, &snapshot);
4632        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
4633            spec: spec.clone(),
4634            enabled: snapshot.lock().unwrap_or_else(|p| p.into_inner()).enabled,
4635            health: runtime.health.clone(),
4636            register_timeout: runtime.register_timeout,
4637            restart_policy: runtime.restart_policy,
4638        }));
4639        let stderr_ring = Arc::clone(&runtime.stderr_ring);
4640        let terminal_ring = Arc::clone(&runtime.terminal_ring);
4641        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
4642        let (tx, rx) = mpsc::channel(4);
4643        let module_id = spec.module_id.clone();
4644        let module = SupervisedModule {
4645            inner: Arc::new(SupervisedModuleInner {
4646                module_id: module_id.clone(),
4647                registry: Arc::clone(&self.registry),
4648                snapshot,
4649                configuration,
4650                stderr_ring,
4651                terminal_ring,
4652                commands: tx,
4653                monitor: Mutex::new(None),
4654                effective_drain_timeout,
4655                provenance_probe: self.provenance_probe.clone(),
4656                retained: runtime.child_roster.retained_view(),
4657            }),
4658        };
4659        // Identity gates and the configured mark must precede the roster entry,
4660        // so a client never sees a module without its admission policy.
4661        if let Some(supervisor_handle) = &self.supervisor_handle {
4662            supervisor_handle.insert(module.clone());
4663        }
4664        (module, rx)
4665    }
4666
4667    fn start_monitor(
4668        &self,
4669        spec: ModuleSpec,
4670        runtime: SupervisorRuntimeConfig,
4671        module: &SupervisedModule,
4672        child: Option<SupervisedChild>,
4673        commands: mpsc::Receiver<SupervisorCommand>,
4674    ) {
4675        let monitor = tokio::spawn(supervise_loop(
4676            spec,
4677            runtime,
4678            Arc::clone(&self.registry),
4679            Arc::clone(&self.process_liveness),
4680            Arc::clone(&module.inner.snapshot),
4681            child,
4682            commands,
4683        ));
4684        *module
4685            .inner
4686            .monitor
4687            .lock()
4688            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
4689    }
4690}
4691
4692impl Default for Supervisor {
4693    fn default() -> Self {
4694        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
4695    }
4696}
4697
4698/// Handle to one supervised child process.
4699#[derive(Clone)]
4700pub struct SupervisedModule {
4701    inner: Arc<SupervisedModuleInner>,
4702}
4703
4704struct SupervisedModuleInner {
4705    retained: crate::child_roster::RetainedView,
4706    module_id: String,
4707    registry: Arc<Registry>,
4708    snapshot: SharedSnapshot,
4709    configuration: Arc<Mutex<SupervisedConfiguration>>,
4710    stderr_ring: Arc<Mutex<StderrRing>>,
4711    terminal_ring: Arc<Mutex<TerminalRing>>,
4712    commands: mpsc::Sender<SupervisorCommand>,
4713    monitor: Mutex<Option<JoinHandle<()>>>,
4714    effective_drain_timeout: Arc<Mutex<Duration>>,
4715    provenance_probe: ExecutableIdentityProbe,
4716}
4717
4718impl fmt::Debug for SupervisedModule {
4719    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4720        f.debug_struct("SupervisedModule")
4721            .field("module_id", &self.inner.module_id)
4722            .field("status", &self.status())
4723            .finish_non_exhaustive()
4724    }
4725}
4726
4727impl SupervisedModule {
4728    pub(crate) fn check_approved_start_failure(
4729        &self,
4730        generation: &crate::config_gate::runtime::ApprovalIdentity,
4731    ) -> Result<bool, SuperviseError> {
4732        let state = lock_snapshot(&self.inner.snapshot)?;
4733        if let Some((failed_generation, detail)) = &state.approved_start_failure {
4734            if failed_generation == generation {
4735                return Err(SuperviseError::ReloadFailed {
4736                    module_id: self.inner.module_id.clone(),
4737                    reason: detail.clone(),
4738                });
4739            }
4740        }
4741        Ok(state.approved_start_failure.is_some())
4742    }
4743
4744    pub(crate) fn has_approved_start_failure(
4745        &self,
4746        identity: &crate::config_gate::runtime::ApprovalIdentity,
4747    ) -> bool {
4748        lock_snapshot(&self.inner.snapshot).is_ok_and(|state| {
4749            state
4750                .approved_start_failure
4751                .as_ref()
4752                .is_some_and(|(failed, _)| failed == identity)
4753        })
4754    }
4755
4756    pub(crate) fn needs_approved_gate_restart(
4757        &self,
4758        approval: &crate::config_gate::runtime::ApprovalIdentity,
4759    ) -> bool {
4760        lock_snapshot(&self.inner.snapshot)
4761            .is_ok_and(|state| state.needs_approved_gate_restart(approval))
4762    }
4763
4764    #[cfg(all(test, unix))]
4765    pub(crate) fn config_hold_origin_for_test(
4766        &self,
4767    ) -> (bool, Option<crate::config_gate::runtime::ApprovalIdentity>) {
4768        let state = lock_snapshot(&self.inner.snapshot).unwrap();
4769        (
4770            state.config_hold_daemon_initiated,
4771            state.config_hold_approval.clone(),
4772        )
4773    }
4774
4775    pub(crate) async fn config_policy_probe(
4776        &self,
4777        connection: crate::ConnectionId,
4778        generation: u64,
4779        attempt: u64,
4780    ) -> Result<(), SuperviseError> {
4781        self.inner
4782            .commands
4783            .send(SupervisorCommand::ConfigPolicyProbe {
4784                connection,
4785                generation,
4786                attempt,
4787            })
4788            .await
4789            .map_err(|_| SuperviseError::CommandClosed {
4790                module_id: self.inner.module_id.clone(),
4791            })
4792    }
4793    pub(crate) async fn config_exit_hold(&self) -> Result<(), SuperviseError> {
4794        let (reply, result) = oneshot::channel();
4795        self.inner
4796            .commands
4797            .send(SupervisorCommand::ConfigExitHold { reply })
4798            .await
4799            .map_err(|_| SuperviseError::CommandClosed {
4800                module_id: self.module_id().to_string(),
4801            })?;
4802        result.await.map_err(|_| SuperviseError::CommandClosed {
4803            module_id: self.module_id().to_string(),
4804        })?
4805    }
4806    pub(crate) async fn config_delivery_failed(
4807        &self,
4808        connection: crate::ConnectionId,
4809        hold: bool,
4810    ) -> Result<(), SuperviseError> {
4811        self.inner
4812            .commands
4813            .send(SupervisorCommand::ConfigDeliveryFailed { connection, hold })
4814            .await
4815            .map_err(|_| SuperviseError::CommandClosed {
4816                module_id: self.module_id().to_string(),
4817            })
4818    }
4819
4820    pub(crate) async fn config_apply_restart(&self, generation: u64) -> Result<(), SuperviseError> {
4821        self.inner
4822            .commands
4823            .send(SupervisorCommand::ConfigApplyRestart {
4824                generation,
4825                supersedes_swap: false,
4826            })
4827            .await
4828            .map_err(|_| SuperviseError::CommandClosed {
4829                module_id: self.inner.module_id.clone(),
4830            })
4831    }
4832    pub fn module_id(&self) -> &str {
4833        &self.inner.module_id
4834    }
4835
4836    /// Test-only: put one probe miss on the streak, the way
4837    /// `handle_health_probe_failure` does, so tests can assert what a later
4838    /// event does to the streak without driving the whole probe loop.
4839    #[cfg(test)]
4840    pub(crate) fn record_health_probe_failure_for_test(
4841        &self,
4842        detail: &str,
4843    ) -> Result<(), SuperviseError> {
4844        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
4845            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
4846            state.health.detail = Some(detail.to_string());
4847        })
4848    }
4849
4850    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
4851        Ok(lock_snapshot(&self.inner.snapshot)?.state)
4852    }
4853
4854    pub(crate) fn registration_end_reason(
4855        &self,
4856    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
4857        let snapshot = lock_snapshot(&self.inner.snapshot)?;
4858        Ok(match snapshot.state {
4859            ModuleState::Draining if snapshot.draining_to_replace => {
4860                Some(RegistrationEndReason::SupervisorRestart)
4861            }
4862            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
4863            ModuleState::Restarting if snapshot.draining_to_replace => {
4864                Some(RegistrationEndReason::SupervisorRestart)
4865            }
4866            _ => None,
4867        })
4868    }
4869
4870    /// The module's retained stderr, newest lines last.
4871    ///
4872    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
4873    /// module, `supervisor.list` renders every module, and putting it in the
4874    /// shared snapshot would make each status read carry a payload almost nobody
4875    /// asked for. Callers that want the text ask for it.
4876    pub fn stderr_tail(
4877        &self,
4878        max_lines: Option<usize>,
4879        max_bytes: Option<usize>,
4880    ) -> StderrTailSnapshot {
4881        self.inner
4882            .stderr_ring
4883            .lock()
4884            .unwrap_or_else(|poisoned| poisoned.into_inner())
4885            .snapshot(max_lines, max_bytes)
4886    }
4887
4888    /// The module's bounded terminal history, oldest retained exit first.
4889    ///
4890    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
4891    /// daemon whose in-memory history was necessarily reset.
4892    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
4893        self.inner
4894            .terminal_ring
4895            .lock()
4896            .unwrap_or_else(|poisoned| poisoned.into_inner())
4897            .snapshot()
4898    }
4899
4900    /// Retained observations from the current ring and all journal generations.
4901    ///
4902    /// Blocking: this reads the journal files. Async callers use
4903    /// `Self::read_durable_terminal_history`.
4904    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
4905        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
4906    }
4907
4908    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
4909    /// read (up to every retained generation) never occupies a runtime worker.
4910    /// Fails only if the blocking task could not finish (runtime shutdown or a
4911    /// panic in the read).
4912    pub(crate) async fn read_durable_terminal_history(
4913        &self,
4914    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
4915        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
4916        let module_id = self.inner.module_id.clone();
4917        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
4918            .await
4919    }
4920
4921    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
4922        self.status_with_snapshot_lock(&self.inner.snapshot, None)
4923            .map(|(status, _)| status)
4924    }
4925
4926    pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
4927        let state = lock_snapshot(&self.inner.snapshot)?;
4928        Ok(state
4929            .failed_retry
4930            .pending
4931            .filter(|_| state.state == ModuleState::Failed && state.enabled)
4932            .map(|retry| {
4933                unix_ms_now().saturating_add(
4934                    u64::try_from(
4935                        retry
4936                            .deadline
4937                            .saturating_duration_since(Instant::now())
4938                            .as_millis(),
4939                    )
4940                    .unwrap_or(u64::MAX),
4941                )
4942            }))
4943    }
4944
4945    #[cfg(all(test, unix))]
4946    pub(crate) fn retry_deadline_for_test(&self) -> Option<Instant> {
4947        lock_snapshot(&self.inner.snapshot)
4948            .unwrap()
4949            .failed_retry
4950            .pending
4951            .map(|retry| retry.deadline)
4952    }
4953
4954    pub(crate) fn record_deliberate_severance(
4955        &self,
4956        identity: ProcessIdentity,
4957    ) -> Result<bool, SuperviseError> {
4958        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4959        if snapshot.pid != Some(identity.pid)
4960            || snapshot.process_start_time != Some(identity.start_time)
4961        {
4962            return Ok(false);
4963        }
4964        snapshot.deliberate_severance = Some(identity);
4965        Ok(true)
4966    }
4967
4968    /// Read status for a channel-0 renderer and report a contended snapshot lock.
4969    ///
4970    /// Internal supervision callers use [`Self::status`] so writer-side machinery
4971    /// does not produce reader-observability logs.
4972    pub(crate) fn status_for_control(
4973        &self,
4974        caller: &'static str,
4975    ) -> Result<ModuleStatus, SuperviseError> {
4976        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
4977            .map(|(status, _)| status)
4978    }
4979
4980    fn status_with_snapshot_lock(
4981        &self,
4982        snapshot: &SharedSnapshot,
4983        caller: Option<&'static str>,
4984    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
4985        let restart_policy = self
4986            .inner
4987            .configuration
4988            .lock()
4989            .map_err(|_| SuperviseError::StatePoisoned {
4990                module_id: Some(self.inner.module_id.clone()),
4991            })?
4992            .restart_policy;
4993        let mut guard = match caller {
4994            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
4995            None => lock_snapshot(snapshot)?,
4996        };
4997        // Read the budget through the pruning path so a reader sees the same
4998        // in-window count the restart decision would use, not a stale total.
4999        let restart_count = guard.crash_restarts_in_window(restart_policy.window, Instant::now());
5000        let snapshot = guard.clone();
5001        drop(guard);
5002        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
5003            SuperviseError::StatePoisoned {
5004                module_id: Some(self.inner.module_id.clone()),
5005            }
5006        })?;
5007        let registration_active = self
5008            .inner
5009            .registry
5010            .get_module(&self.inner.module_id)
5011            .map_err(SuperviseError::Registry)?
5012            .is_some();
5013        let protocol = snapshot
5014            .spawned_protocol
5015            .unwrap_or(self.declared_protocol()?);
5016        let running_process =
5017            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
5018        // Registration is the difference between the two protocols and the only
5019        // one: a subc module that has not registered cannot serve a request even
5020        // though its process is up, and a `none` module never registers at all,
5021        // so requiring it there would pin `live` to false for the whole life of
5022        // a perfectly healthy process.
5023        let live = match protocol {
5024            ModuleProtocol::Subc => running_process && registration_active,
5025            ModuleProtocol::None => running_process,
5026        };
5027        // Read only the retained cache: a status read must not resend signals,
5028        // rewrite the record, or keep the containment resource alive.
5029        let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
5030        let carry_configured = self
5031            .inner
5032            .configuration
5033            .lock()
5034            .map_err(|_| SuperviseError::StatePoisoned {
5035                module_id: Some(self.inner.module_id.clone()),
5036            })?
5037            .spec
5038            .carry_containment;
5039        let recovery_block = carries_containment(
5040            carry_configured,
5041            self.inner
5042                .retained
5043                .has_carried_container(&self.inner.module_id),
5044        )
5045        .then(|| self.inner.retained.recovery_block())
5046        .flatten();
5047        let launch_nonce_source = observed_launch_nonce_source(&snapshot);
5048        let pid = snapshot.reported_pid();
5049        let mut health = snapshot.health;
5050        if let Some(detail) = recovery_block
5051            .as_ref()
5052            .or_else(|| retained.as_ref().map(|(_, detail)| detail))
5053        {
5054            health.detail = Some(match health.detail {
5055                Some(previous) => format!("{previous}; {detail}"),
5056                None => detail.clone(),
5057            });
5058        }
5059
5060        Ok((
5061            ModuleStatus {
5062                admission_refused: snapshot.admission_refused,
5063                legacy_registration_spawn: snapshot.legacy_registration_spawn,
5064                registration_declaration: snapshot.registration_declaration.clone(),
5065                stop_reason: snapshot.stop_reason,
5066                waiting_for_previous_pid: retained
5067                    .as_ref()
5068                    .map(|(pid, _)| *pid)
5069                    .filter(|pid| *pid > 0)
5070                    .or(snapshot.waiting_for_previous_pid),
5071                waiting_for_previous_detail: recovery_block
5072                    .or_else(|| retained.map(|(_, detail)| detail))
5073                    .or(snapshot.waiting_for_previous_detail),
5074                module_id: self.inner.module_id.clone(),
5075                launch_nonce_source,
5076                state: snapshot.state,
5077                enabled: snapshot.enabled,
5078                process_alive: snapshot.process_alive,
5079                registration_active,
5080                protocol,
5081                live,
5082                restart_count,
5083                lifetime_restarts: snapshot.lifetime_restarts,
5084                spawn_generation: snapshot.spawn_generation,
5085                swap_outcome: snapshot.swap_outcome,
5086                max_restarts: restart_policy.max_restarts,
5087                restart_window: restart_policy.window,
5088                drain_timeout,
5089                restart_backoff: restart_policy.backoff,
5090                restart_max_backoff: restart_policy.max_backoff,
5091                pid,
5092                spawned_at_ms: snapshot.spawned_at_ms,
5093                spawned_from: snapshot.spawned_from,
5094                launch_spec_digest: snapshot.launch_spec_digest,
5095                process_start_time: snapshot.process_start_time,
5096                last_exit: snapshot.last_exit,
5097                health,
5098            },
5099            snapshot.spawned_file_identity.clone(),
5100        ))
5101    }
5102
5103    #[cfg(test)]
5104    pub(crate) fn hold_snapshot_for_test(
5105        &self,
5106        acquired: std::sync::mpsc::Sender<()>,
5107        hold: Duration,
5108    ) -> std::thread::JoinHandle<()> {
5109        let snapshot = Arc::clone(&self.inner.snapshot);
5110        std::thread::spawn(move || {
5111            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
5112            acquired
5113                .send(())
5114                .expect("test receiver waits for snapshot lock");
5115            std::thread::sleep(hold);
5116        })
5117    }
5118
5119    /// The status and the running-image check for `supervisor.provenance`,
5120    /// taken from one status read. The exec acknowledgement can land between
5121    /// two separate reads, and the reply would then pair "no pid yet" with an
5122    /// image observed after the module started, which describes no single
5123    /// moment.
5124    pub(crate) async fn status_and_running_image_agreement(
5125        &self,
5126    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
5127        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
5128        let image = self
5129            .inner
5130            .provenance_probe
5131            .observe(
5132                status.pid,
5133                status.spawned_from.as_deref(),
5134                identity,
5135                status.process_start_time,
5136            )
5137            .await;
5138        Ok((status, image))
5139    }
5140
5141    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
5142        let snapshot = match lock_snapshot(&self.inner.snapshot) {
5143            Ok(snapshot) => snapshot.clone(),
5144            Err(_) => {
5145                return subc_control::RunningImageAgreement::Unavailable {
5146                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
5147                };
5148            }
5149        };
5150        self.inner
5151            .provenance_probe
5152            .observe(
5153                snapshot.reported_pid(),
5154                snapshot.spawned_from.as_deref(),
5155                snapshot.spawned_file_identity.clone(),
5156                snapshot.process_start_time,
5157            )
5158            .await
5159    }
5160
5161    /// Memory and CPU time of the module's current process, read now. Only the
5162    /// process the supervisor spawned is read, not processes it has started.
5163    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
5164        #[cfg(windows)]
5165        {
5166            let image = lock_snapshot(&self.inner.snapshot)
5167                .ok()
5168                .and_then(|snapshot| {
5169                    snapshot
5170                        .spawned_file_identity
5171                        .as_ref()
5172                        .and_then(|identity| identity.windows_image.clone())
5173                });
5174            match image {
5175                Some(image) => {
5176                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
5177                }
5178                None => subc_control::ChildResourceUsage::Unavailable {
5179                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
5180                },
5181            }
5182        }
5183        #[cfg(not(windows))]
5184        {
5185            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
5186                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
5187                Err(_) => {
5188                    return subc_control::ChildResourceUsage::Unavailable {
5189                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
5190                    }
5191                }
5192            };
5193            crate::child_resources::read(pid, start_time)
5194        }
5195    }
5196
5197    #[cfg(all(test, any(target_os = "linux", windows)))]
5198    pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
5199        self.inner
5200            .snapshot
5201            .try_lock()
5202            .map(|snapshot| snapshot.subtree_target.clone())
5203            .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
5204    }
5205
5206    #[cfg(test)]
5207    pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
5208        self.inner.snapshot.lock().unwrap().subtree_target = target;
5209    }
5210
5211    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
5212        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
5213        Ok(match snapshot.state {
5214            ModuleState::Restarting => true,
5215            ModuleState::Failed | ModuleState::Disabled => false,
5216            _ if snapshot.enabled && snapshot.respawn_pending => true,
5217            _ => daemon_will_restart(
5218                &mut snapshot,
5219                &self
5220                    .inner
5221                    .configuration
5222                    .lock()
5223                    .map_err(|_| SuperviseError::StatePoisoned {
5224                        module_id: Some(self.inner.module_id.clone()),
5225                    })?
5226                    .restart_policy,
5227                Instant::now(),
5228            ),
5229        })
5230    }
5231
5232    #[cfg(test)]
5233    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
5234        self.is_warming_with_snapshot_lock(None)
5235    }
5236
5237    pub(crate) fn is_warming_for_control(
5238        &self,
5239        caller: &'static str,
5240    ) -> Result<bool, SuperviseError> {
5241        self.is_warming_with_snapshot_lock(Some(caller))
5242    }
5243
5244    fn is_warming_with_snapshot_lock(
5245        &self,
5246        caller: Option<&'static str>,
5247    ) -> Result<bool, SuperviseError> {
5248        let snapshot = match caller {
5249            Some(caller) => {
5250                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
5251            }
5252            None => lock_snapshot(&self.inner.snapshot)?,
5253        }
5254        .clone();
5255        Ok(matches!(
5256            snapshot.state,
5257            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
5258        ))
5259    }
5260
5261    /// Drain the module and stop monitoring it.
5262    pub async fn drain(&self) -> Result<(), SuperviseError> {
5263        self.stop().await
5264    }
5265
5266    pub(crate) async fn retire_requested(
5267        &self,
5268        requester: Option<Principal>,
5269    ) -> Result<(), SuperviseError> {
5270        match self.state()? {
5271            ModuleState::Stopped => return Ok(()),
5272            ModuleState::Failed
5273                if lock_snapshot(&self.inner.snapshot)?
5274                    .failed_retry
5275                    .pending
5276                    .is_none() =>
5277            {
5278                return Ok(())
5279            }
5280            ModuleState::Failed
5281            | ModuleState::Starting
5282            | ModuleState::Running
5283            | ModuleState::Unresponsive
5284            | ModuleState::Restarting
5285            | ModuleState::Draining
5286            | ModuleState::Disabled => {}
5287        }
5288
5289        let (reply_tx, reply_rx) = oneshot::channel();
5290        self.inner
5291            .commands
5292            .send(SupervisorCommand::Retire {
5293                requester,
5294                reply: reply_tx,
5295            })
5296            .await
5297            .map_err(|_| SuperviseError::CommandClosed {
5298                module_id: self.inner.module_id.clone(),
5299            })?;
5300        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5301            module_id: self.inner.module_id.clone(),
5302        })?
5303    }
5304
5305    pub async fn stop(&self) -> Result<(), SuperviseError> {
5306        self.stop_requested(None).await
5307    }
5308
5309    pub(crate) async fn stop_requested(
5310        &self,
5311        requester: Option<Principal>,
5312    ) -> Result<(), SuperviseError> {
5313        match self.state()? {
5314            ModuleState::Stopped => return Ok(()),
5315            ModuleState::Failed
5316                if lock_snapshot(&self.inner.snapshot)?
5317                    .failed_retry
5318                    .pending
5319                    .is_none() =>
5320            {
5321                return Ok(())
5322            }
5323            ModuleState::Failed
5324            | ModuleState::Starting
5325            | ModuleState::Running
5326            | ModuleState::Unresponsive
5327            | ModuleState::Restarting
5328            | ModuleState::Draining
5329            | ModuleState::Disabled => {}
5330        }
5331
5332        let (reply_tx, reply_rx) = oneshot::channel();
5333        self.inner
5334            .commands
5335            .send(SupervisorCommand::Drain {
5336                requester,
5337                reply: reply_tx,
5338            })
5339            .await
5340            .map_err(|_| SuperviseError::CommandClosed {
5341                module_id: self.inner.module_id.clone(),
5342            })?;
5343        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5344            module_id: self.inner.module_id.clone(),
5345        })?
5346    }
5347
5348    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
5349        self.restart_requested(drain_timeout_ms, None).await
5350    }
5351
5352    pub(crate) async fn restart_requested(
5353        &self,
5354        drain_timeout_ms: Option<u64>,
5355        requester: Option<Principal>,
5356    ) -> Result<(), SuperviseError> {
5357        let queued_at = Instant::now();
5358        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
5359        let (reply_tx, reply_rx) = oneshot::channel();
5360        self.inner
5361            .commands
5362            .send(SupervisorCommand::Restart {
5363                requester,
5364                drain_timeout_ms,
5365                received_at_generation,
5366                queued_at,
5367                reply: reply_tx,
5368            })
5369            .await
5370            .map_err(|_| SuperviseError::CommandClosed {
5371                module_id: self.inner.module_id.clone(),
5372            })?;
5373        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5374            module_id: self.inner.module_id.clone(),
5375        })?
5376    }
5377
5378    /// Blue/green restart: see `SupervisorCommand::Swap` and the
5379    /// `supervisor_swap` module. Returns once the swap has cut over (the old
5380    /// process then drains in the background of the supervise loop), has been
5381    /// superseded by an approved configuration restart, or has failed. Both
5382    /// supersession and failure leave the old process serving until replacement.
5383    pub async fn swap(
5384        &self,
5385        ready_timeout: Option<Duration>,
5386    ) -> Result<SwapOutcome, SuperviseError> {
5387        self.swap_requested(ready_timeout, None).await
5388    }
5389
5390    pub(crate) async fn swap_requested(
5391        &self,
5392        ready_timeout: Option<Duration>,
5393        requester: Option<Principal>,
5394    ) -> Result<SwapOutcome, SuperviseError> {
5395        let (reply_tx, reply_rx) = oneshot::channel();
5396        self.inner
5397            .commands
5398            .send(SupervisorCommand::Swap {
5399                requester,
5400                ready_timeout,
5401                reply: reply_tx,
5402            })
5403            .await
5404            .map_err(|_| SuperviseError::CommandClosed {
5405                module_id: self.inner.module_id.clone(),
5406            })?;
5407        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5408            module_id: self.inner.module_id.clone(),
5409        })?
5410    }
5411
5412    pub async fn reload(&self) -> Result<(), SuperviseError> {
5413        self.reload_requested(None).await
5414    }
5415
5416    pub(crate) async fn reload_requested(
5417        &self,
5418        requester: Option<Principal>,
5419    ) -> Result<(), SuperviseError> {
5420        let (reply_tx, reply_rx) = oneshot::channel();
5421        self.inner
5422            .commands
5423            .send(SupervisorCommand::Reload {
5424                requester,
5425                reply: reply_tx,
5426            })
5427            .await
5428            .map_err(|_| SuperviseError::CommandClosed {
5429                module_id: self.inner.module_id.clone(),
5430            })?;
5431        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5432            module_id: self.inner.module_id.clone(),
5433        })?
5434    }
5435
5436    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
5437        self.set_enabled_requested(enabled, None).await
5438    }
5439
5440    pub(crate) async fn set_enabled_requested(
5441        &self,
5442        enabled: bool,
5443        requester: Option<Principal>,
5444    ) -> Result<bool, SuperviseError> {
5445        self.set_enabled_with_budget_policy(enabled, requester, true)
5446            .await
5447    }
5448
5449    pub(crate) async fn release_config_hold(&self) -> Result<bool, SuperviseError> {
5450        self.set_enabled_with_budget_policy(true, None, false).await
5451    }
5452
5453    #[cfg(all(test, unix))]
5454    pub(crate) fn seed_crash_budget_for_test(&self, count: u32) {
5455        let policy = self.inner.configuration.lock().unwrap().restart_policy;
5456        let mut state = self.inner.snapshot.lock().unwrap();
5457        for _ in 0..count {
5458            state.record_crash_restart(&policy, Instant::now());
5459        }
5460    }
5461
5462    async fn set_enabled_with_budget_policy(
5463        &self,
5464        enabled: bool,
5465        requester: Option<Principal>,
5466        reset_budget: bool,
5467    ) -> Result<bool, SuperviseError> {
5468        let (reply_tx, reply_rx) = oneshot::channel();
5469        self.inner
5470            .commands
5471            .send(SupervisorCommand::SetEnabled {
5472                requester,
5473                enabled,
5474                reset_budget,
5475                reply: reply_tx,
5476            })
5477            .await
5478            .map_err(|_| SuperviseError::CommandClosed {
5479                module_id: self.inner.module_id.clone(),
5480            })?;
5481        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5482            module_id: self.inner.module_id.clone(),
5483        })?
5484    }
5485
5486    /// The current process's protocol, or the configured protocol when down.
5487    /// A rescan stores the next launch spec without changing how an existing
5488    /// process registers, serves routes, is probed, or exits.
5489    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
5490        let configured = self
5491            .inner
5492            .configuration
5493            .lock()
5494            .map_err(|_| SuperviseError::StatePoisoned {
5495                module_id: Some(self.inner.module_id.clone()),
5496            })?
5497            .spec
5498            .protocol;
5499        let state = lock_snapshot(&self.inner.snapshot)?;
5500        Ok(state.spawned_protocol.unwrap_or(configured))
5501    }
5502
5503    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
5504        let configuration =
5505            self.inner
5506                .configuration
5507                .lock()
5508                .map_err(|_| SuperviseError::StatePoisoned {
5509                    module_id: Some(self.inner.module_id.clone()),
5510                })?;
5511        Ok((configuration.spec.clone(), configuration.health.clone()))
5512    }
5513
5514    pub(crate) fn configured_register_timeout(&self) -> Result<Duration, SuperviseError> {
5515        self.inner
5516            .configuration
5517            .lock()
5518            .map(|configuration| configuration.register_timeout)
5519            .map_err(|_| SuperviseError::StatePoisoned {
5520                module_id: Some(self.inner.module_id.clone()),
5521            })
5522    }
5523
5524    pub(crate) fn configured_enabled(&self) -> Result<bool, SuperviseError> {
5525        self.inner
5526            .configuration
5527            .lock()
5528            .map(|configuration| configuration.enabled)
5529            .map_err(|_| SuperviseError::StatePoisoned {
5530                module_id: Some(self.inner.module_id.clone()),
5531            })
5532    }
5533
5534    /// Replace this module's launch spec, keeping its health and drain policy,
5535    /// the way a rescan does for a changed config entry. The running process is
5536    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
5537    #[cfg(any(test, feature = "test-support"))]
5538    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
5539        let (_, health) = self.configuration()?;
5540        let drain_timeout_ms = u64::try_from(
5541            self.inner
5542                .effective_drain_timeout
5543                .lock()
5544                .unwrap_or_else(|poisoned| poisoned.into_inner())
5545                .as_millis(),
5546        )
5547        .ok();
5548        self.update_configuration(spec, health, drain_timeout_ms)
5549            .await
5550    }
5551
5552    pub(crate) async fn reconcile_approved_configuration(
5553        &self,
5554        approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5555    ) -> Result<(), SuperviseError> {
5556        let entry = approved
5557            .module_entry(&self.inner.module_id)
5558            .map_err(|error| SuperviseError::InvalidSpec {
5559                reason: error.to_string(),
5560            })?;
5561        validate_spec(&entry.module_spec())?;
5562        let (reply, received) = oneshot::channel();
5563        self.inner
5564            .commands
5565            .send(SupervisorCommand::ApprovedConfiguration { approved, reply })
5566            .await
5567            .map_err(|_| SuperviseError::CommandClosed {
5568                module_id: self.inner.module_id.clone(),
5569            })?;
5570        received.await.map_err(|_| SuperviseError::CommandClosed {
5571            module_id: self.inner.module_id.clone(),
5572        })??;
5573        let mut configuration =
5574            self.inner
5575                .configuration
5576                .lock()
5577                .map_err(|_| SuperviseError::StatePoisoned {
5578                    module_id: Some(self.inner.module_id.clone()),
5579                })?;
5580        configuration.spec = entry.module_spec();
5581        configuration.enabled = entry.enabled;
5582        configuration.health = entry.health;
5583        configuration.register_timeout = entry.register_timeout;
5584        configuration.restart_policy = entry.restart;
5585        Ok(())
5586    }
5587
5588    pub(crate) async fn update_configuration(
5589        &self,
5590        spec: ModuleSpec,
5591        health: HealthConfig,
5592        drain_timeout_ms: Option<u64>,
5593    ) -> Result<(), SuperviseError> {
5594        if spec.module_id != self.inner.module_id {
5595            return Err(SuperviseError::InvalidSpec {
5596                reason: "a supervised module's module_id cannot be changed".to_string(),
5597            });
5598        }
5599        validate_spec(&spec)?;
5600        let (reply_tx, reply_rx) = oneshot::channel();
5601        self.inner
5602            .commands
5603            .send(SupervisorCommand::UpdateConfiguration {
5604                spec: spec.clone(),
5605                health: health.clone(),
5606                drain_timeout_ms,
5607                reply: reply_tx,
5608            })
5609            .await
5610            .map_err(|_| SuperviseError::CommandClosed {
5611                module_id: self.inner.module_id.clone(),
5612            })?;
5613        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5614            module_id: self.inner.module_id.clone(),
5615        })??;
5616        let mut configuration =
5617            self.inner
5618                .configuration
5619                .lock()
5620                .map_err(|_| SuperviseError::StatePoisoned {
5621                    module_id: Some(self.inner.module_id.clone()),
5622                })?;
5623        configuration.spec = spec;
5624        configuration.health = health;
5625        Ok(())
5626    }
5627}
5628
5629impl Drop for SupervisedModuleInner {
5630    fn drop(&mut self) {
5631        let Ok(mut monitor) = self.monitor.lock() else {
5632            return;
5633        };
5634        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
5635            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
5636                state.state = ModuleState::Stopped;
5637                clear_current_process_facts(state);
5638            });
5639            monitor.abort();
5640        }
5641        let _ = monitor.take();
5642    }
5643}
5644
5645#[derive(Debug)]
5646enum SupervisorCommand {
5647    ApprovedConfiguration {
5648        approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5649        reply: oneshot::Sender<Result<(), SuperviseError>>,
5650    },
5651    ConfigPolicyProbe {
5652        connection: crate::ConnectionId,
5653        generation: u64,
5654        attempt: u64,
5655    },
5656    ConfigApplyRestart {
5657        generation: u64,
5658        supersedes_swap: bool,
5659    },
5660    ConfigExitHold {
5661        reply: oneshot::Sender<Result<(), SuperviseError>>,
5662    },
5663    ConfigDeliveryFailed {
5664        connection: crate::ConnectionId,
5665        hold: bool,
5666    },
5667    Drain {
5668        requester: Option<Principal>,
5669        reply: oneshot::Sender<Result<(), SuperviseError>>,
5670    },
5671    Retire {
5672        requester: Option<Principal>,
5673        reply: oneshot::Sender<Result<(), SuperviseError>>,
5674    },
5675    Restart {
5676        requester: Option<Principal>,
5677        /// Operator override for this one restart's drain budget, in ms. `None`
5678        /// uses the module's configured/default budget; `Some(0)` cuts
5679        /// immediately (wedge bounce: a stuck request never settles, so
5680        /// waiting only delays recovery).
5681        drain_timeout_ms: Option<u64>,
5682        /// The module's `spawn_generation` when the request was received, before
5683        /// it waited in the command queue. A queued restart whose module has
5684        /// since spawned a newer process is already satisfied (see the handler).
5685        received_at_generation: u64,
5686        /// When the request entered the command queue, so the handler can log
5687        /// how long it waited behind the loop's other work.
5688        queued_at: Instant,
5689        reply: oneshot::Sender<Result<(), SuperviseError>>,
5690    },
5691    Reload {
5692        requester: Option<Principal>,
5693        reply: oneshot::Sender<Result<(), SuperviseError>>,
5694    },
5695    SetEnabled {
5696        reset_budget: bool,
5697        requester: Option<Principal>,
5698        enabled: bool,
5699        reply: oneshot::Sender<Result<bool, SuperviseError>>,
5700    },
5701    UpdateConfiguration {
5702        spec: ModuleSpec,
5703        health: HealthConfig,
5704        /// Per-module drain override from the new config; `None` re-resolves to
5705        /// the supervisor-wide default.
5706        drain_timeout_ms: Option<u64>,
5707        reply: oneshot::Sender<Result<(), SuperviseError>>,
5708    },
5709    Swap {
5710        requester: Option<Principal>,
5711        /// How long the candidate may take to register and declare itself
5712        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
5713        ready_timeout: Option<Duration>,
5714        /// Answered at cutover or failure; the incumbent's drain follows.
5715        reply: oneshot::Sender<Result<SwapOutcome, SuperviseError>>,
5716    },
5717}
5718
5719/// Successful daemon-local swap disposition. Supersession leaves the incumbent
5720/// serving until an approved replacement starts with the latest configured
5721/// command; the legacy control acknowledgment does not carry this disposition.
5722#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5723pub enum SwapOutcome {
5724    Cutover,
5725    SupersededByConfigRestart,
5726}
5727
5728impl SwapOutcome {
5729    pub fn as_str(self) -> &'static str {
5730        match self {
5731            Self::Cutover => "cutover",
5732            Self::SupersededByConfigRestart => "superseded_by_config_restart",
5733        }
5734    }
5735}
5736
5737#[derive(Debug)]
5738pub enum SuperviseError {
5739    ConfigHeld {
5740        module_id: String,
5741        code: String,
5742        detail: String,
5743    },
5744    ConfigDeliveryDeferred {
5745        module_id: String,
5746    },
5747    InvalidSpec {
5748        reason: String,
5749    },
5750    Spawn {
5751        program: PathBuf,
5752        source: io::Error,
5753        cgroup_path: Option<PathBuf>,
5754    },
5755    Cgroup {
5756        module_id: String,
5757        source: io::Error,
5758    },
5759    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
5760    /// than spawn a reserved module without its identity binding.
5761    LaunchNonce {
5762        reason: String,
5763    },
5764    Wait {
5765        module_id: String,
5766        source: io::Error,
5767    },
5768    Kill {
5769        module_id: String,
5770        source: io::Error,
5771    },
5772    Forwarding(ForwardingError),
5773    Registry(RegistryError),
5774    ReloadUnavailable {
5775        module_id: String,
5776        reason: String,
5777    },
5778    /// An operator restart/reload was requested for a module that is currently
5779    /// disabled. Restart/reload cycle a *running* module; a disabled module must
5780    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
5781    /// by a restart, so these commands are rejected instead of re-enabling it.
5782    Disabled {
5783        module_id: String,
5784    },
5785    ReloadFailed {
5786        module_id: String,
5787        reason: String,
5788    },
5789    RegistrationStillActive {
5790        module_id: String,
5791        waited: Duration,
5792    },
5793    StatePoisoned {
5794        module_id: Option<String>,
5795    },
5796    CommandClosed {
5797        module_id: String,
5798    },
5799    /// A restart or reload arrived while a swap's candidate was warming. The
5800    /// swap owns the module until it cuts over or fails; a stop or disable
5801    /// would have aborted it instead.
5802    SwapInProgress {
5803        module_id: String,
5804    },
5805    /// A swap was refused before anything was spawned.
5806    SwapRefused {
5807        module_id: String,
5808        reason: SwapRefusal,
5809    },
5810    /// A swap spawned a candidate and gave up on it. The candidate has been
5811    /// killed and its slot freed; the incumbent was left serving and was never
5812    /// drained, except in the one `CutoverLost` case described on that arm.
5813    SwapFailed {
5814        module_id: String,
5815        arm: SwapFailureArm,
5816        detail: String,
5817        /// How the candidate exited, when it exited on its own before the
5818        /// supervisor gave up on it.
5819        candidate_exit: Option<ExitReport>,
5820    },
5821}
5822
5823/// Why a swap was refused before a candidate was spawned.
5824#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5825pub enum SwapRefusal {
5826    /// The module's config does not declare `overlap: "safe"`.
5827    OverlapExclusive,
5828    /// The module is not registered, so there is no incumbent to keep serving
5829    /// and nothing a swap would improve on; a plain restart is the tool.
5830    NotRegistered,
5831    /// The module does not speak the subc wire, so a candidate could never
5832    /// register or declare itself ready.
5833    ProtocolNone,
5834    /// The supervisor lacks the forwarding table (to cut routes over) or the
5835    /// shared handle (to admit the candidate's HELLO) that a swap needs.
5836    NotConfigured,
5837    /// A swap is already open for this module.
5838    AlreadySwapping,
5839}
5840
5841impl SwapRefusal {
5842    pub fn as_str(self) -> &'static str {
5843        match self {
5844            Self::OverlapExclusive => "overlap_exclusive",
5845            Self::NotRegistered => "not_registered",
5846            Self::ProtocolNone => "protocol_none",
5847            Self::NotConfigured => "not_configured",
5848            Self::AlreadySwapping => "already_swapping",
5849        }
5850    }
5851}
5852
5853/// Which failure arm ended a swap. Every arm but one leaves the incumbent
5854/// serving and undrained; see `CutoverLost`.
5855#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5856pub enum SwapFailureArm {
5857    /// The candidate process could not be started.
5858    SpawnFailed,
5859    /// The candidate did not register within the readiness budget.
5860    NeverRegistered,
5861    /// The candidate registered but did not declare itself ready in time.
5862    NeverReady,
5863    /// The candidate exited before cutover.
5864    CandidateExited,
5865    /// The candidate declared itself ready but failed its health probe.
5866    CandidateUnhealthy,
5867    /// An operator stop, disable or retire arrived while the candidate warmed.
5868    /// The candidate was killed and the operator's command then carried out on
5869    /// the incumbent.
5870    Interrupted,
5871    /// The candidate's connection closed at the moment of cutover. If it
5872    /// closed before forwarding moved, the incumbent is untouched. If it closed
5873    /// between the forwarding and registry halves of cutover, forwarding can no
5874    /// longer route to the incumbent, so the module is restarted plainly.
5875    CutoverLost,
5876}
5877
5878impl SwapFailureArm {
5879    pub fn as_str(self) -> &'static str {
5880        match self {
5881            Self::SpawnFailed => "spawn_failed",
5882            Self::NeverRegistered => "never_registered",
5883            Self::NeverReady => "never_ready",
5884            Self::CandidateExited => "candidate_exited",
5885            Self::CandidateUnhealthy => "candidate_unhealthy",
5886            Self::Interrupted => "interrupted",
5887            Self::CutoverLost => "cutover_lost",
5888        }
5889    }
5890}
5891
5892impl SuperviseError {
5893    pub(crate) fn is_config_gate_error(&self) -> bool {
5894        match self {
5895            Self::ConfigHeld { .. } => true,
5896            Self::Spawn { source, .. } => source
5897                .get_ref()
5898                .is_some_and(|cause| cause.is::<crate::config_gate::GateError>()),
5899            _ => false,
5900        }
5901    }
5902}
5903
5904impl fmt::Display for SuperviseError {
5905    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5906        match self {
5907            Self::ConfigHeld { module_id, code, detail } => write!(f, "{code}: module '{module_id}' held: {detail}"),
5908            Self::ConfigDeliveryDeferred { module_id } => write!(f, "config_gated: module '{module_id}' is waiting for its exited delivery target's failure record"),
5909            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
5910            Self::Spawn {
5911                program,
5912                source,
5913                cgroup_path: Some(cgroup_path),
5914            } => write!(
5915                f,
5916                "failed to place module in cgroup '{}' while spawning '{}': {source}",
5917                cgroup_path.display(),
5918                program.display()
5919            ),
5920            Self::Spawn {
5921                program,
5922                source,
5923                cgroup_path: None,
5924            } => write!(
5925                f,
5926                "failed to spawn module '{}': {source}",
5927                program.display()
5928            ),
5929            Self::Cgroup { module_id, source } => {
5930                write!(
5931                    f,
5932                    "failed to prepare cgroup for module '{module_id}': {source}"
5933                )
5934            }
5935            Self::LaunchNonce { reason } => {
5936                write!(
5937                    f,
5938                    "failed to generate reserved-module launch nonce: {reason}"
5939                )
5940            }
5941            Self::Wait { module_id, source } => {
5942                write!(f, "failed to wait for module '{module_id}': {source}")
5943            }
5944            Self::Kill { module_id, source } => {
5945                write!(f, "failed to kill module '{module_id}': {source}")
5946            }
5947            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
5948            Self::Registry(err) => write!(f, "registry error: {err}"),
5949            Self::ReloadUnavailable { module_id, reason } => {
5950                write!(f, "reload unavailable for module '{module_id}': {reason}")
5951            }
5952            Self::Disabled { module_id } => {
5953                write!(
5954                    f,
5955                    "module '{module_id}' is disabled; enable it before restart or reload"
5956                )
5957            }
5958            Self::ReloadFailed { module_id, reason } => {
5959                write!(f, "reload failed for module '{module_id}': {reason}")
5960            }
5961            Self::RegistrationStillActive { module_id, waited } => write!(
5962                f,
5963                "module '{module_id}' registration remained active after waiting {waited:?}"
5964            ),
5965            Self::StatePoisoned { module_id } => match module_id {
5966                Some(module_id) => {
5967                    write!(f, "supervisor state for module '{module_id}' was poisoned")
5968                }
5969                None => write!(f, "supervisor state was poisoned"),
5970            },
5971            Self::CommandClosed { module_id } => {
5972                write!(
5973                    f,
5974                    "supervisor command channel for module '{module_id}' is closed"
5975                )
5976            }
5977            Self::SwapInProgress { module_id } => write!(
5978                f,
5979                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
5980            ),
5981            Self::SwapRefused { module_id, reason } => match reason {
5982                SwapRefusal::OverlapExclusive => write!(
5983                    f,
5984                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
5985                ),
5986                SwapRefusal::NotRegistered => write!(
5987                    f,
5988                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
5989                ),
5990                SwapRefusal::ProtocolNone => write!(
5991                    f,
5992                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
5993                ),
5994                SwapRefusal::NotConfigured => write!(
5995                    f,
5996                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
5997                ),
5998                SwapRefusal::AlreadySwapping => {
5999                    write!(f, "module '{module_id}' is already being swapped")
6000                }
6001            },
6002            Self::SwapFailed {
6003                module_id,
6004                arm,
6005                detail,
6006                ..
6007            } => write!(
6008                f,
6009                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
6010                arm.as_str()
6011            ),
6012        }
6013    }
6014}
6015
6016impl Error for SuperviseError {
6017    fn source(&self) -> Option<&(dyn Error + 'static)> {
6018        match self {
6019            Self::Spawn { source, .. }
6020            | Self::Cgroup { source, .. }
6021            | Self::Wait { source, .. }
6022            | Self::Kill { source, .. } => Some(source),
6023            Self::Forwarding(err) => Some(err),
6024            Self::Registry(err) => Some(err),
6025            Self::LaunchNonce { .. }
6026            | Self::ConfigDeliveryDeferred { .. }
6027            | Self::ConfigHeld { .. }
6028            | Self::InvalidSpec { .. }
6029            | Self::ReloadUnavailable { .. }
6030            | Self::Disabled { .. }
6031            | Self::ReloadFailed { .. }
6032            | Self::RegistrationStillActive { .. }
6033            | Self::StatePoisoned { .. }
6034            | Self::CommandClosed { .. }
6035            | Self::SwapInProgress { .. }
6036            | Self::SwapRefused { .. }
6037            | Self::SwapFailed { .. } => None,
6038        }
6039    }
6040}
6041
6042pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
6043    if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
6044        return Err(SuperviseError::InvalidSpec {
6045            reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
6046        });
6047    }
6048    if spec.module_id.trim().is_empty() {
6049        return Err(SuperviseError::InvalidSpec {
6050            reason: "module_id must not be empty".to_string(),
6051        });
6052    }
6053
6054    Ok(())
6055}
6056
6057#[derive(Debug, Default)]
6058struct HealthProbeRuntime {
6059    spawn_generation: Option<u64>,
6060    configured_health: Option<HealthConfig>,
6061    registered_connection: Option<crate::ConnectionId>,
6062    advertised: bool,
6063    next_probe_at: Option<Instant>,
6064    probe_index: u64,
6065}
6066
6067fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
6068    lock_snapshot(snapshot)
6069        .ok()
6070        .and_then(|state| state.spawned_protocol)
6071        .unwrap_or(spec.protocol)
6072}
6073
6074impl HealthProbeRuntime {
6075    fn refresh_registration(
6076        &mut self,
6077        spec: &ModuleSpec,
6078        runtime: &SupervisorRuntimeConfig,
6079        registry: &Registry,
6080        snapshot: &SharedSnapshot,
6081    ) {
6082        let generation = lock_snapshot(snapshot)
6083            .ok()
6084            .map(|state| state.spawn_generation);
6085        if self.spawn_generation != generation {
6086            // A replacement cannot reuse a deadline that is already due for its
6087            // predecessor. HTTP children get a full cadence to start; wire
6088            // children still wait for their own health-advertising HELLO.
6089            *self = Self {
6090                spawn_generation: generation,
6091                ..Self::default()
6092            };
6093        }
6094        if self.configured_health.as_ref() != Some(&runtime.health) {
6095            self.configured_health = Some(runtime.health.clone());
6096            self.next_probe_at = None;
6097            self.registered_connection = None;
6098            self.probe_index = 0;
6099        }
6100        // A non-wire process never registers. Only an explicitly configured
6101        // HTTP endpoint can arm its health probe; an absent HELLO is not a
6102        // health failure for that kind of process.
6103        if running_protocol(spec, snapshot) == ModuleProtocol::None {
6104            self.registered_connection = None;
6105            self.advertised = runtime.health.http.is_some();
6106            if !self.advertised {
6107                self.next_probe_at = None;
6108                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6109                    let unknown = ModuleHealthStatus::default();
6110                    if state.health != unknown {
6111                        state.health = unknown;
6112                        state.last_probe_evidence = None;
6113                    }
6114                });
6115            } else if self.next_probe_at.is_none() {
6116                self.next_probe_at = Some(
6117                    Instant::now()
6118                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6119                );
6120            }
6121            return;
6122        }
6123
6124        let registration = match registry.get_module(&spec.module_id) {
6125            Ok(registration) => registration,
6126            Err(err) => {
6127                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
6128                self.advertised = false;
6129                self.next_probe_at = None;
6130                return;
6131            }
6132        };
6133
6134        let Some(registration) = registration else {
6135            self.registered_connection = None;
6136            self.advertised = false;
6137            self.next_probe_at = None;
6138            return;
6139        };
6140
6141        let advertised = registration
6142            .control_ops
6143            .iter()
6144            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
6145        if !advertised {
6146            self.registered_connection = Some(registration.connection_id);
6147            self.advertised = false;
6148            self.next_probe_at = None;
6149            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6150                if state.health.status != SupervisorHealthStatus::Unknown
6151                    || state.health.consecutive_failures != 0
6152                    || state.health.last_probe_ms.is_some()
6153                    || state.health.detail.is_some()
6154                    || state.health.metrics.is_some()
6155                {
6156                    state.health.status = SupervisorHealthStatus::Unknown;
6157                    state.last_probe_evidence = None;
6158                    state.health.consecutive_failures = 0;
6159                    state.health.last_probe_ms = None;
6160                    state.health.detail = None;
6161                    state.health.metrics = None;
6162                }
6163            });
6164            return;
6165        }
6166
6167        let reregistered = self.registered_connection != Some(registration.connection_id);
6168        self.registered_connection = Some(registration.connection_id);
6169        self.advertised = true;
6170        if reregistered || self.next_probe_at.is_none() {
6171            self.probe_index = 0;
6172            self.next_probe_at = Some(
6173                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6174            );
6175            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6176                state.health.status = SupervisorHealthStatus::Unknown;
6177                state.last_probe_evidence = None;
6178                state.health.consecutive_failures = 0;
6179                state.health.detail = None;
6180                state.health.metrics = None;
6181            });
6182        }
6183    }
6184
6185    fn wake_at(&self) -> Option<Instant> {
6186        if !self.advertised {
6187            return None;
6188        }
6189        self.next_probe_at
6190    }
6191
6192    fn due(&self) -> bool {
6193        self.advertised
6194            && self
6195                .next_probe_at
6196                .is_some_and(|next| Instant::now() >= next)
6197    }
6198
6199    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
6200        self.probe_index = self.probe_index.wrapping_add(1);
6201        self.next_probe_at = Some(
6202            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
6203        );
6204    }
6205}
6206
6207/// Evidence is separate from diagnostic prose so host starvation can hold only
6208/// silence-based restarts. A missing lane is still actionable during overload;
6209/// a timeout can instead mean the host has not scheduled a healthy module.
6210#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6211enum HealthProbeEvidence {
6212    /// The module's control lane is gone. Proof of death.
6213    LaneDead,
6214    /// No reply within the deadline. Proves nothing about the module's state.
6215    NoAnswer,
6216    /// The module replied, but not with a usable health report. Proves it is alive.
6217    BadAnswer,
6218    /// The daemon could not ask. Says nothing about the module.
6219    Misconfigured,
6220}
6221
6222#[derive(Debug)]
6223struct HealthProbeError {
6224    evidence: HealthProbeEvidence,
6225    message: String,
6226}
6227
6228impl HealthProbeError {
6229    fn lane_dead(message: impl Into<String>) -> Self {
6230        Self::with(HealthProbeEvidence::LaneDead, message)
6231    }
6232
6233    fn no_answer(message: impl Into<String>) -> Self {
6234        Self::with(HealthProbeEvidence::NoAnswer, message)
6235    }
6236
6237    fn bad_answer(message: impl Into<String>) -> Self {
6238        Self::with(HealthProbeEvidence::BadAnswer, message)
6239    }
6240
6241    fn misconfigured(message: impl Into<String>) -> Self {
6242        Self::with(HealthProbeEvidence::Misconfigured, message)
6243    }
6244
6245    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
6246        Self {
6247            evidence,
6248            message: message.into(),
6249        }
6250    }
6251
6252    /// Whether this observation is proof the module cannot serve.
6253    ///
6254    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
6255    /// variant that fires under CPU starvation, and treating it as proof is the
6256    /// defect this enum exists to make impossible to reintroduce silently.
6257    ///
6258    /// Outside a host-overload hold, silence still reaches the normal failure
6259    /// threshold so an isolated, genuinely wedged module can recover.
6260    #[allow(dead_code)]
6261    fn is_proof_of_death(&self) -> bool {
6262        matches!(self.evidence, HealthProbeEvidence::LaneDead)
6263    }
6264
6265    /// Short stable label for logs and the health snapshot.
6266    ///
6267    /// An operator reading `ck health` currently cannot tell "the module is gone"
6268    /// from "the module did not answer in five seconds", because both render as
6269    /// prose in the same field. These labels are what make the two
6270    /// distinguishable at a glance, and they are what a later restart-policy
6271    /// change will be argued from.
6272    fn label(&self) -> &'static str {
6273        match self.evidence {
6274            HealthProbeEvidence::LaneDead => "lane-dead",
6275            HealthProbeEvidence::NoAnswer => "no-answer",
6276            HealthProbeEvidence::BadAnswer => "bad-answer",
6277            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
6278        }
6279    }
6280}
6281
6282impl fmt::Display for HealthProbeError {
6283    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
6284        f.write_str(&self.message)
6285    }
6286}
6287
6288async fn run_health_probe_cycle(
6289    spec: &ModuleSpec,
6290    runtime: &SupervisorRuntimeConfig,
6291    registry: &Registry,
6292    process_liveness: &SupervisorProcessLiveness,
6293    snapshot: &SharedSnapshot,
6294    child: &mut Option<SupervisedChild>,
6295) {
6296    let now_ms = unix_ms_now();
6297    let delivery_probe = match &runtime.config_delivery {
6298        Some(delivery) => {
6299            delivery
6300                .health_probe(&spec.module_id, runtime.forwarding.as_deref(), registry)
6301                .await
6302        }
6303        None => None,
6304    };
6305    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
6306        .then_some(runtime.health.http.as_deref())
6307        .flatten();
6308    let result = match http {
6309        Some(url) => probe_http_health(url, runtime.health.deadline)
6310            .await
6311            .map(|report| (report, None)),
6312        None => probe_module_health_from(&spec.module_id, runtime, None)
6313            .await
6314            .map(|(report, probed)| (report, Some(probed))),
6315    };
6316    match result {
6317        Ok((report, probed)) => {
6318            handle_health_report(
6319                spec,
6320                runtime,
6321                registry,
6322                process_liveness,
6323                snapshot,
6324                child,
6325                report,
6326                now_ms,
6327                delivery_probe,
6328                probed,
6329            )
6330            .await;
6331        }
6332        Err(err) => {
6333            if http.is_some() {
6334                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6335                    state.health.status = SupervisorHealthStatus::Failing;
6336                });
6337            }
6338            handle_health_probe_failure(
6339                spec,
6340                runtime,
6341                registry,
6342                process_liveness,
6343                snapshot,
6344                child,
6345                err,
6346                now_ms,
6347                delivery_probe,
6348            )
6349            .await;
6350        }
6351    }
6352}
6353
6354pub(crate) struct HttpProbeTarget<'a> {
6355    address: std::net::SocketAddr,
6356    localhost: bool,
6357    authority: &'a str,
6358    path: String,
6359}
6360
6361/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
6362/// or TLS. A URL cannot turn a local health check into an outbound connection.
6363pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
6364    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
6365        return Err("must not contain whitespace, controls, or a fragment".into());
6366    }
6367    let rest = url
6368        .strip_prefix("http://")
6369        .ok_or("must use plain http://")?;
6370    let split = rest.find(['/', '?']).unwrap_or(rest.len());
6371    let (authority, suffix) = rest.split_at(split);
6372    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
6373        ("::1", rest)
6374    } else {
6375        let split = authority.find(':').unwrap_or(authority.len());
6376        authority.split_at(split)
6377    };
6378    let ip: std::net::IpAddr = match host {
6379        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
6380        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
6381        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
6382    };
6383    let port = if port.is_empty() {
6384        80
6385    } else {
6386        port.strip_prefix(':')
6387            .and_then(|p| p.parse::<u16>().ok())
6388            .filter(|p| *p > 0)
6389            .ok_or("must have a valid nonzero TCP port")?
6390    };
6391    let path = if suffix.is_empty() {
6392        "/".into()
6393    } else if suffix.starts_with('?') {
6394        format!("/{suffix}")
6395    } else {
6396        suffix.into()
6397    };
6398    Ok(HttpProbeTarget {
6399        address: std::net::SocketAddr::new(ip, port),
6400        localhost: host == "localhost",
6401        authority,
6402        path,
6403    })
6404}
6405
6406async fn probe_http_health(
6407    url: &str,
6408    deadline: Duration,
6409) -> Result<HealthReport, HealthProbeError> {
6410    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
6411    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
6412    // Keep partial diagnostics outside the timed future so cancellation does
6413    // not discard a status line or body bytes already received.
6414    let mut response_status = String::new();
6415    let mut body = Vec::new();
6416    let probe = async {
6417        // Resolve localhost ourselves so a hosts-file override cannot turn
6418        // this into an outbound request, while IPv6-only local servers work.
6419        let connection = match tokio::net::TcpStream::connect(target.address).await {
6420            Err(_) if target.localhost => {
6421                tokio::net::TcpStream::connect((
6422                    std::net::Ipv6Addr::LOCALHOST,
6423                    target.address.port(),
6424                ))
6425                .await
6426            }
6427            result => result,
6428        };
6429        let mut stream = connection.map_err(|error| {
6430            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
6431        })?;
6432        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
6433            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
6434        let mut reader = BufReader::new(stream);
6435        let mut budget = 16 * 1024;
6436        let status = http_line(&mut reader, &mut budget).await?;
6437        let mut words = status.split_ascii_whitespace();
6438        let version = words.next();
6439        let code = words
6440            .next()
6441            .filter(|word| word.len() == 3)
6442            .and_then(|word| word.parse::<u16>().ok());
6443        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
6444            || !code.is_some_and(|code| (100..600).contains(&code))
6445        {
6446            return Err(HealthProbeError::bad_answer(format!(
6447                "invalid HTTP status: {status}"
6448            )));
6449        }
6450        let code = code.expect("validated status code");
6451        response_status = status.clone();
6452        let mut length = None;
6453        let mut chunked = false;
6454        loop {
6455            let line = http_line(&mut reader, &mut budget).await?;
6456            if line.is_empty() {
6457                break;
6458            }
6459            if let Some((name, value)) = line.split_once(':') {
6460                if name.eq_ignore_ascii_case("content-length") {
6461                    length = Some(value.trim().parse::<u64>().map_err(|_| {
6462                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
6463                    })?);
6464                } else if name.eq_ignore_ascii_case("transfer-encoding") {
6465                    chunked = value.trim().eq_ignore_ascii_case("chunked");
6466                }
6467            }
6468        }
6469        if chunked {
6470            while body.len() < 200 {
6471                let line = http_line(&mut reader, &mut budget).await?;
6472                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
6473                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
6474                if size == 0 {
6475                    break;
6476                }
6477                let count = size.min((200 - body.len()) as u64) as usize;
6478                let start = body.len();
6479                (&mut reader)
6480                    .take(count as u64)
6481                    .read_to_end(&mut body)
6482                    .await
6483                    .map_err(|error| {
6484                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6485                    })?;
6486                if body.len() - start != count {
6487                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
6488                }
6489                if size > count as u64 || body.len() == 200 {
6490                    break;
6491                }
6492                if !http_line(&mut reader, &mut budget).await?.is_empty() {
6493                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
6494                }
6495            }
6496        } else {
6497            reader
6498                .take(length.unwrap_or(200).min(200))
6499                .read_to_end(&mut body)
6500                .await
6501                .map_err(|error| {
6502                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6503                })?;
6504        }
6505        if (200..300).contains(&code) {
6506            Ok(HealthReport::ok())
6507        } else {
6508            Err(HealthProbeError::bad_answer(
6509                "HTTP health endpoint returned non-2xx",
6510            ))
6511        }
6512    };
6513    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
6514        Err(HealthProbeError::no_answer(format!(
6515            "HTTP probe timed out after {deadline:?}"
6516        )))
6517    });
6518    if let Err(error) = &mut result {
6519        if !response_status.is_empty() {
6520            error.message = format!(
6521                "{}; {response_status}: {}",
6522                error.message,
6523                String::from_utf8_lossy(&body)
6524            );
6525        }
6526    }
6527    result
6528}
6529
6530async fn http_line(
6531    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
6532    remaining: &mut usize,
6533) -> Result<String, HealthProbeError> {
6534    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
6535    let mut line = Vec::new();
6536    (&mut *reader)
6537        .take(*remaining as u64)
6538        .read_until(b'\n', &mut line)
6539        .await
6540        .map_err(|error| {
6541            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
6542        })?;
6543    *remaining -= line.len();
6544    if !line.ends_with(b"\r\n") {
6545        return Err(HealthProbeError::bad_answer(
6546            "HTTP headers are incomplete or exceed 16 KiB",
6547        ));
6548    }
6549    line.truncate(line.len() - 2);
6550    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
6551}
6552
6553async fn probe_module_health(
6554    module_id: &str,
6555    runtime: &SupervisorRuntimeConfig,
6556    drain_deadline: Option<Instant>,
6557) -> Result<HealthReport, HealthProbeError> {
6558    probe_module_health_from(module_id, runtime, drain_deadline)
6559        .await
6560        .map(|(report, _)| report)
6561}
6562
6563/// [`probe_module_health`], also returning the connection whose registration
6564/// answered, so its answer is credited to that registration and no other.
6565async fn probe_module_health_from(
6566    module_id: &str,
6567    runtime: &SupervisorRuntimeConfig,
6568    drain_deadline: Option<Instant>,
6569) -> Result<(HealthReport, ConnectionId), HealthProbeError> {
6570    let Some(forwarding) = runtime.forwarding.as_ref() else {
6571        return Err(HealthProbeError::misconfigured(
6572            "supervisor was not configured with a forwarding table",
6573        ));
6574    };
6575    let probe_started_at = Instant::now();
6576    let mut deadline = probe_started_at + runtime.health.deadline;
6577    if let Some(drain_deadline) = drain_deadline {
6578        deadline = deadline.min(drain_deadline);
6579    }
6580    let pending = if drain_deadline.is_some() {
6581        forwarding.begin_drain_health_probe_rpc_for(
6582            module_id,
6583            MODULE_CONTROL_OP_HEALTH_CHECK,
6584            probe_started_at,
6585            deadline,
6586        )
6587    } else {
6588        forwarding.begin_health_probe_rpc_for(
6589            module_id,
6590            MODULE_CONTROL_OP_HEALTH_CHECK,
6591            probe_started_at,
6592            deadline,
6593        )
6594    }
6595    .map_err(|err| {
6596        // The endpoint is not registered, so there is no live control lane to
6597        // ask. That is the module being absent, not slow.
6598        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6599    })?;
6600    let probed = pending.endpoint.connection_id;
6601    await_health_probe(forwarding, pending, deadline, runtime.health.deadline)
6602        .await
6603        .map(|report| (report, probed))
6604}
6605
6606/// [`probe_module_health`] for one endpoint rather than the id's active one.
6607///
6608/// A swap probes two processes that no by-id lookup reaches: its candidate
6609/// before cutover, and its superseded incumbent (for busy gauges) while the
6610/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
6611/// bounds the by-id drain probe.
6612async fn probe_endpoint_health(
6613    endpoint: crate::ModuleEndpointId,
6614    runtime: &SupervisorRuntimeConfig,
6615    deadline_cap: Option<Instant>,
6616) -> Result<HealthReport, HealthProbeError> {
6617    let Some(forwarding) = runtime.forwarding.as_ref() else {
6618        return Err(HealthProbeError::misconfigured(
6619            "supervisor was not configured with a forwarding table",
6620        ));
6621    };
6622    let probe_started_at = Instant::now();
6623    let mut deadline = probe_started_at + runtime.health.deadline;
6624    if let Some(cap) = deadline_cap {
6625        deadline = deadline.min(cap);
6626    }
6627    let pending = forwarding
6628        .begin_endpoint_health_probe_rpc_for(
6629            endpoint,
6630            MODULE_CONTROL_OP_HEALTH_CHECK,
6631            probe_started_at,
6632            deadline,
6633        )
6634        .map_err(|err| {
6635            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6636        })?;
6637    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
6638}
6639
6640/// Send a begun health probe and classify its answer.
6641async fn await_health_probe(
6642    forwarding: &ForwardingTable,
6643    pending: PendingModuleControlRpc,
6644    deadline: Instant,
6645    probe_budget: Duration,
6646) -> Result<HealthReport, HealthProbeError> {
6647    let PendingModuleControlRpc {
6648        endpoint,
6649        module_sink,
6650        negotiated_ver,
6651        corr,
6652        receiver,
6653    } = pending;
6654    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6655        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
6656    })?;
6657    let frame = Frame::build_with_version(
6658        negotiated_ver,
6659        FrameType::Request,
6660        control_flags(),
6661        0,
6662        0,
6663        corr,
6664        body,
6665    )
6666    .map_err(|err| {
6667        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
6668    })?;
6669
6670    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
6671    // blocks waiting for capacity when the module's egress queue is full, and an
6672    // unbounded await here freezes the whole supervision actor (it stops polling
6673    // Child::wait and supervisor commands), making the module unrecoverable
6674    // in-band. On timeout the probe fails like any transport failure.
6675    match timeout_at(deadline, module_sink.send(frame)).await {
6676        Ok(Ok(())) => {}
6677        Ok(Err(err)) => {
6678            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6679            // A closed sink means the module's egress channel is gone -- the
6680            // receiving half is dropped when its connection tears down. Proof.
6681            return Err(HealthProbeError::lane_dead(format!(
6682                "failed to send health.check: {err}"
6683            )));
6684        }
6685        Err(_elapsed) => {
6686            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6687            // A full egress queue means the module is not draining its socket, which
6688            // is consistent with a wedged module AND with one whose reader is merely
6689            // starved. Silence, not proof.
6690            return Err(HealthProbeError::no_answer(
6691                "health.check send timed out before enqueue (module egress full)",
6692            ));
6693        }
6694    }
6695
6696    match timeout_at(deadline, receiver).await {
6697        Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => Err(HealthProbeError::bad_answer(
6698            "health.check returned an unexpected config response",
6699        )),
6700        // Each arm records WHAT WAS OBSERVED. Four of them are the module
6701        // demonstrably answering -- rejected, non-health, malformed, wrong op --
6702        // and those prove it is alive even though the probe failed.
6703        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6704            response.health_report().ok_or_else(|| {
6705                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
6706            })
6707        }
6708        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
6709            format!("health.check rejected: {}", body.message),
6710        )),
6711        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6712            Err(HealthProbeError::lane_dead(message))
6713        }
6714        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6715            Err(HealthProbeError::bad_answer(message))
6716        }
6717        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6718            Err(HealthProbeError::bad_answer(format!(
6719                "expected module-control op '{expected}', got '{actual}'"
6720            )))
6721        }
6722        // A reply that crosses the deadline before this waiter observes it is
6723        // still proof of life. The forwarding path records its end-to-end latency
6724        // before delivering this classification.
6725        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
6726            "module answered health.check after its daemon deadline",
6727        )),
6728        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
6729            "health.check waiter was canceled before the module responded",
6730        )),
6731        Err(_) => {
6732            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
6733            Err(HealthProbeError::no_answer(format!(
6734                "module did not answer health.check within {probe_budget:?}"
6735            )))
6736        }
6737    }
6738}
6739
6740/// Act on a health probe's answer. `probed` is the connection whose
6741/// registration answered (`None` for an HTTP probe, which reaches no
6742/// registration); only that registration's spawn is credited with readiness.
6743#[allow(clippy::too_many_arguments)]
6744async fn handle_health_report(
6745    spec: &ModuleSpec,
6746    runtime: &SupervisorRuntimeConfig,
6747    registry: &Registry,
6748    process_liveness: &SupervisorProcessLiveness,
6749    snapshot: &SharedSnapshot,
6750    child: &mut Option<SupervisedChild>,
6751    report: HealthReport,
6752    now_ms: u64,
6753    delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6754    probed: Option<ConnectionId>,
6755) {
6756    let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6757        (Some(delivery), Some(probe)) => {
6758            delivery
6759                .health_result(
6760                    &spec.module_id,
6761                    runtime.forwarding.as_deref(),
6762                    registry,
6763                    probe,
6764                    report.status == HealthStatus::Ok,
6765                )
6766                .await
6767        }
6768        _ => false,
6769    };
6770    if let (Some(delivery), Some(spawn)) = (
6771        &runtime.config_delivery,
6772        child
6773            .as_ref()
6774            .and_then(|child| child.config_delivery.as_ref()),
6775    ) {
6776        delivery
6777            .mark_readiness(
6778                &spec.module_id,
6779                runtime.forwarding.as_deref(),
6780                probed,
6781                spawn,
6782                report.status == HealthStatus::Ok,
6783            )
6784            .await;
6785    }
6786    let status = supervisor_health_status(report.status);
6787    let detail = report.detail.clone();
6788    let metrics = truncate_health_metrics(report.metrics);
6789    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6790        state.health.status = status;
6791        state.last_probe_evidence = None;
6792        state.health.last_probe_ms = Some(now_ms);
6793        state.health.detail = detail.clone();
6794        state.health.metrics = metrics.clone();
6795        state.health.consecutive_failures = 0;
6796        state.failed_retry.observe_health(
6797            report.status == HealthStatus::Ok,
6798            runtime.restart_policy.window,
6799        );
6800    });
6801
6802    if delivery_failed {
6803        return;
6804    }
6805    let action = match report.status {
6806        HealthStatus::Ok => return,
6807        HealthStatus::Degraded => runtime.health.on_degraded,
6808        HealthStatus::Failing => runtime.health.on_failing,
6809    };
6810    apply_l3_health_action(
6811        spec,
6812        runtime,
6813        registry,
6814        process_liveness,
6815        snapshot,
6816        child,
6817        status,
6818        detail.as_deref(),
6819        action,
6820        now_ms,
6821    )
6822    .await;
6823}
6824
6825#[allow(clippy::too_many_arguments)]
6826async fn handle_health_probe_failure(
6827    spec: &ModuleSpec,
6828    runtime: &SupervisorRuntimeConfig,
6829    registry: &Registry,
6830    process_liveness: &SupervisorProcessLiveness,
6831    snapshot: &SharedSnapshot,
6832    child: &mut Option<SupervisedChild>,
6833    err: HealthProbeError,
6834    now_ms: u64,
6835    delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6836) {
6837    let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6838        (Some(delivery), Some(probe)) => {
6839            delivery
6840                .health_result(
6841                    &spec.module_id,
6842                    runtime.forwarding.as_deref(),
6843                    registry,
6844                    probe,
6845                    false,
6846                )
6847                .await
6848        }
6849        _ => false,
6850    };
6851    let threshold = runtime.health.failure_threshold.max(1);
6852    let held = if err.evidence == HealthProbeEvidence::NoAnswer {
6853        runtime.host_health.record_late_module(&spec.module_id)
6854    } else {
6855        false
6856    };
6857    let mut failures = 0;
6858    // Carry the evidence class into the operator-visible detail. Without it,
6859    // "module did not answer within 5s" and "the control lane is gone" are two
6860    // prose strings in the same field, and the reader has to know the codebase to
6861    // tell which one is proof of anything.
6862    let detail = if held {
6863        format!("[{}] {err}; host overloaded", err.label())
6864    } else {
6865        format!("[{}] {err}", err.label())
6866    };
6867    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6868        state.last_probe_evidence = Some(err.evidence);
6869        state
6870            .failed_retry
6871            .observe_health(false, runtime.restart_policy.window);
6872        // A failed wire probe invalidates the last report, even before the
6873        // restart threshold. HTTP probes already mark failures as Failing.
6874        if held {
6875            state.health.status = SupervisorHealthStatus::Degraded;
6876        } else if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
6877            state.health.status = SupervisorHealthStatus::Unknown;
6878        }
6879        state.health.last_probe_ms = Some(now_ms);
6880        // Held silence breaks the streak: after the host recovers, only new
6881        // failures count. Replaying pre-overload misses would restart immediately.
6882        state.health.consecutive_failures = if held {
6883            0
6884        } else {
6885            state.health.consecutive_failures.saturating_add(1)
6886        };
6887        state.health.detail = Some(detail.clone());
6888        state.health.metrics = None;
6889        failures = state.health.consecutive_failures;
6890    });
6891
6892    if delivery_failed {
6893        return;
6894    }
6895    if held || failures < threshold {
6896        warn!(
6897            module_id = %spec.module_id,
6898            consecutive_failures = failures,
6899            threshold,
6900            evidence = err.label(),
6901            detail = %detail,
6902            "health.check probe failed"
6903        );
6904        return;
6905    }
6906
6907    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6908        state.state = ModuleState::Unresponsive;
6909        state.health.status = SupervisorHealthStatus::Unresponsive;
6910    });
6911    // The evidence class is logged at the kill site because this is the line an
6912    // operator reads after an unexplained restart. A streak of `no-answer` under
6913    // machine load is the known false-positive shape; a `lane-dead` is not.
6914    if runtime.health.critical {
6915        error!(
6916            module_id = %spec.module_id,
6917            status = "unresponsive",
6918            evidence = err.label(),
6919            detail = %detail,
6920            "critical module health alert"
6921        );
6922    } else {
6923        warn!(
6924            module_id = %spec.module_id,
6925            status = "unresponsive",
6926            evidence = err.label(),
6927            detail = %detail,
6928            "module health threshold breached"
6929        );
6930    }
6931    if let Err(err) = health_restart_child(
6932        spec,
6933        runtime,
6934        registry,
6935        process_liveness,
6936        snapshot,
6937        child,
6938        SupervisorHealthStatus::Unresponsive,
6939        Some(&detail),
6940        now_ms,
6941    )
6942    .await
6943    {
6944        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
6945    }
6946}
6947
6948#[allow(clippy::too_many_arguments)]
6949async fn apply_l3_health_action(
6950    spec: &ModuleSpec,
6951    runtime: &SupervisorRuntimeConfig,
6952    registry: &Registry,
6953    process_liveness: &SupervisorProcessLiveness,
6954    snapshot: &SharedSnapshot,
6955    child: &mut Option<SupervisedChild>,
6956    status: SupervisorHealthStatus,
6957    detail: Option<&str>,
6958    action: HealthAction,
6959    now_ms: u64,
6960) {
6961    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
6962    match action {
6963        HealthAction::Report => {
6964            info!(
6965                module_id = %spec.module_id,
6966                status = ?status,
6967                detail,
6968                "module reported non-ok health"
6969            );
6970        }
6971        HealthAction::Alert => {
6972            error!(
6973                module_id = %spec.module_id,
6974                status = ?status,
6975                detail,
6976                "module health alert"
6977            );
6978        }
6979        HealthAction::Restart => {
6980            if let Err(err) = health_restart_child(
6981                spec,
6982                runtime,
6983                registry,
6984                process_liveness,
6985                snapshot,
6986                child,
6987                status,
6988                detail,
6989                now_ms,
6990            )
6991            .await
6992            {
6993                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
6994            }
6995        }
6996    }
6997}
6998
6999#[allow(clippy::too_many_arguments)]
7000async fn health_restart_child(
7001    spec: &ModuleSpec,
7002    runtime: &SupervisorRuntimeConfig,
7003    registry: &Registry,
7004    process_liveness: &SupervisorProcessLiveness,
7005    snapshot: &SharedSnapshot,
7006    child: &mut Option<SupervisedChild>,
7007    status: SupervisorHealthStatus,
7008    detail: Option<&str>,
7009    now_ms: u64,
7010) -> Result<(), SuperviseError> {
7011    budgeted_restart_child(
7012        spec,
7013        runtime,
7014        registry,
7015        process_liveness,
7016        snapshot,
7017        child,
7018        status,
7019        detail,
7020        now_ms,
7021        true,
7022    )
7023    .await
7024}
7025
7026#[allow(clippy::too_many_arguments)]
7027async fn budgeted_restart_child(
7028    spec: &ModuleSpec,
7029    runtime: &SupervisorRuntimeConfig,
7030    registry: &Registry,
7031    process_liveness: &SupervisorProcessLiveness,
7032    snapshot: &SharedSnapshot,
7033    child: &mut Option<SupervisedChild>,
7034    status: SupervisorHealthStatus,
7035    detail: Option<&str>,
7036    now_ms: u64,
7037    health_failure: bool,
7038) -> Result<(), SuperviseError> {
7039    if let Some(error) = runtime
7040        .config_delivery
7041        .as_ref()
7042        .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
7043        .filter(|error| error.code != "config_delivery_deferred")
7044    {
7045        return Err(config_delivery::gate_hold_error(
7046            spec,
7047            error,
7048            runtime.config_delivery.as_ref(),
7049        ));
7050    }
7051    let recovery_reason = if health_failure {
7052        "health_check"
7053    } else {
7054        "configuration_delivery"
7055    };
7056    let (enabled, schedule) = {
7057        let mut state = lock_snapshot(snapshot)?;
7058        let enabled = state.enabled;
7059        let schedule = if enabled {
7060            state.next_crash_restart(&runtime.restart_policy, Instant::now())
7061        } else {
7062            None
7063        };
7064        (enabled, schedule)
7065    };
7066
7067    if !enabled {
7068        return Err(SuperviseError::Disabled {
7069            module_id: spec.module_id.clone(),
7070        });
7071    }
7072
7073    if schedule.is_none() {
7074        if health_failure {
7075            record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
7076        }
7077        error!(
7078            module_id = %spec.module_id,
7079            status = ?status,
7080            detail,
7081            max_restarts = runtime.restart_policy.max_restarts,
7082            window_secs = runtime.restart_policy.window.as_secs(),
7083            reason = %runtime.restart_policy.budget_exhausted_detail(),
7084            recovery_reason,
7085            "automatic restart budget exhausted; marking module failed"
7086        );
7087        let stop_notice = begin_forwarding_drain_if_configured(
7088            spec,
7089            runtime,
7090            registry,
7091            snapshot,
7092            Some(true),
7093            RouteCloseReason::Disable,
7094        )
7095        .await?;
7096        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7097            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
7098        })?;
7099        let result = drain_optional_child(
7100            &spec.module_id,
7101            spec.protocol,
7102            stop_notice,
7103            registry,
7104            runtime.forwarding.as_deref(),
7105            snapshot,
7106            &runtime.terminal_ring,
7107            &runtime.spawn_events,
7108            child,
7109            runtime.drain_timeout,
7110            ModuleState::Failed,
7111            Some(true),
7112            &runtime.failure_context(true),
7113        )
7114        .await;
7115        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7116            if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
7117                state.failed_retry.schedule(&spec.module_id);
7118            }
7119        })?;
7120        result?;
7121        process_liveness.untrack_if_current(&spec.module_id, snapshot);
7122        return Ok(());
7123    }
7124
7125    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
7126    let mut restart_count = 0;
7127    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7128        restart_count = state.crash_restarts.len();
7129        state.state = ModuleState::Unresponsive;
7130        if health_failure {
7131            state.health.status = status;
7132            state.health.last_action = Some(HealthAction::Restart.to_string());
7133            state.health.last_action_ms = Some(now_ms);
7134        }
7135    })?;
7136    warn!(
7137        module_id = %spec.module_id,
7138        status = ?status,
7139        detail,
7140        restart_count,
7141        restart_in_window = schedule.restart_in_window,
7142        delay_ms = schedule.delay.as_millis() as u64,
7143        recovery_reason,
7144        "automatic module restart"
7145    );
7146
7147    let stop_notice = begin_forwarding_drain_if_configured(
7148        spec,
7149        runtime,
7150        registry,
7151        snapshot,
7152        Some(true),
7153        RouteCloseReason::Restart,
7154    )
7155    .await?;
7156    drain_optional_child(
7157        &spec.module_id,
7158        spec.protocol,
7159        stop_notice,
7160        registry,
7161        runtime.forwarding.as_deref(),
7162        snapshot,
7163        &runtime.terminal_ring,
7164        &runtime.spawn_events,
7165        child,
7166        runtime.drain_timeout,
7167        ModuleState::Restarting,
7168        Some(true),
7169        &runtime.failure_context(true),
7170    )
7171    .await?;
7172    schedule_respawn(
7173        runtime,
7174        snapshot,
7175        &spec.module_id,
7176        schedule.delay,
7177        RespawnKind::Spawn,
7178        true,
7179    )
7180}
7181
7182fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
7183    if let Some(reply) = runtime
7184        .deferred_reload_reply
7185        .lock()
7186        .unwrap_or_else(|p| p.into_inner())
7187        .take()
7188    {
7189        let _ = reply.send(Err(SuperviseError::ReloadFailed {
7190            module_id: module_id.to_string(),
7191            reason: reason.to_string(),
7192        }));
7193    }
7194}
7195
7196fn park_config_spawn(
7197    spec: &ModuleSpec,
7198    runtime: &SupervisorRuntimeConfig,
7199    snapshot: &SharedSnapshot,
7200    failure: &FailureContext,
7201    admission_refused: Option<String>,
7202) -> Result<(), SuperviseError> {
7203    *runtime
7204        .scheduled_respawn
7205        .lock()
7206        .unwrap_or_else(|poisoned| poisoned.into_inner()) = None;
7207    if let Some(reply) = runtime
7208        .deferred_reload_reply
7209        .lock()
7210        .unwrap_or_else(|poisoned| poisoned.into_inner())
7211        .take()
7212    {
7213        let _ = reply.send(Err(SuperviseError::ConfigHeld {
7214            module_id: spec.module_id.clone(),
7215            code: "config_gated".into(),
7216            detail: "module is held by configuration admission".into(),
7217        }));
7218    }
7219    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7220        state.admission_refused = admission_refused;
7221        state.config_hold_daemon_initiated = failure.daemon_initiated;
7222        state.config_hold_approval = failure.approval.clone();
7223        state.config_hold_parked = true;
7224        state.state = ModuleState::Stopped;
7225        state.respawn_pending = false;
7226        state.failed_retry.pending = None;
7227        // Parking ends this attempt and discards its failure timestamp. If a
7228        // later release fails, its retry delay starts at that new failure.
7229        state.failed_retry.failed_at = None;
7230        clear_current_process_facts(state);
7231    })
7232}
7233
7234fn try_park_config_attempt(
7235    spec: &ModuleSpec,
7236    runtime: &SupervisorRuntimeConfig,
7237    snapshot: &SharedSnapshot,
7238    failure: &FailureContext,
7239    program_base: Option<&subc_config::Digest>,
7240) -> Result<bool, SuperviseError> {
7241    let result = (|| {
7242        if !lock_snapshot(snapshot)?.enabled {
7243            return Ok(false);
7244        }
7245        #[cfg(test)]
7246        if let Some(delivery) = &runtime.config_delivery {
7247            delivery.0.pause_before_config_park_for_test();
7248        }
7249        let Some(delivery) = &runtime.config_delivery else {
7250            return Ok(false);
7251        };
7252        let parked = delivery
7253            .0
7254            .with_recorded_admission_hold(&spec.module_id, program_base, |refusal| {
7255                park_config_spawn(spec, runtime, snapshot, failure, refusal)
7256            })
7257            .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(delivery)))?;
7258        if let Some(result) = parked {
7259            result?;
7260            Ok(true)
7261        } else {
7262            Ok(false)
7263        }
7264    })();
7265    if let Err(error) = &result {
7266        // A failed park still belongs to the attempt that tried to park. Mark it
7267        // Failed here, before the monitor can use a daemon/operator origin saved
7268        // for a different attempt to decide whether to schedule a slow retry.
7269        fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7270        record_approved_attempt_failure(spec, snapshot, failure, error);
7271    }
7272    result
7273}
7274
7275fn record_approved_attempt_failure(
7276    spec: &ModuleSpec,
7277    snapshot: &SharedSnapshot,
7278    failure: &FailureContext,
7279    error: &SuperviseError,
7280) {
7281    let Some(approval) = &failure.approval else {
7282        return;
7283    };
7284    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7285        if state.enabled
7286            && state.state == ModuleState::Failed
7287            && state
7288                .approved_start_failure
7289                .as_ref()
7290                .is_none_or(|(identity, _)| identity != approval)
7291        {
7292            state.approved_start_failure = Some((approval.clone(), error.to_string()));
7293        }
7294    });
7295}
7296
7297fn hold_config_spawn(
7298    spec: &ModuleSpec,
7299    runtime: &SupervisorRuntimeConfig,
7300    snapshot: &SharedSnapshot,
7301    failure: &FailureContext,
7302) -> Result<(), SuperviseError> {
7303    if !lock_snapshot(snapshot)?.enabled {
7304        return Ok(());
7305    }
7306    if !try_park_config_attempt(spec, runtime, snapshot, failure, None)? {
7307        // This caller consumed an actual start attempt. If its hold ended before
7308        // parking, retain that attempt instead of waiting for a lost release.
7309        defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
7310    }
7311    Ok(())
7312}
7313
7314fn schedule_respawn(
7315    runtime: &SupervisorRuntimeConfig,
7316    snapshot: &SharedSnapshot,
7317    module_id: &str,
7318    delay: Duration,
7319    kind: RespawnKind,
7320    daemon_initiated: bool,
7321) -> Result<(), SuperviseError> {
7322    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
7323    update_snapshot(snapshot, Some(module_id), |state| {
7324        state.respawn_pending = true
7325    })?;
7326    *runtime
7327        .scheduled_respawn
7328        .lock()
7329        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7330        deadline: Instant::now() + delay,
7331        kind,
7332        daemon_initiated,
7333        approval: None,
7334    });
7335    Ok(())
7336}
7337
7338const CONFIG_ADMISSION_RETRY_DELAY: Duration = Duration::from_millis(100);
7339
7340fn defer_config_spawn(
7341    spec: &ModuleSpec,
7342    runtime: &SupervisorRuntimeConfig,
7343    snapshot: &SharedSnapshot,
7344    kind: RespawnKind,
7345    failure: &FailureContext,
7346) -> Result<(), SuperviseError> {
7347    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7348        state.state = ModuleState::Restarting;
7349        state.respawn_pending = true;
7350        clear_current_process_facts(state);
7351    })?;
7352    *runtime
7353        .scheduled_respawn
7354        .lock()
7355        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7356        deadline: Instant::now() + CONFIG_ADMISSION_RETRY_DELAY,
7357        kind,
7358        daemon_initiated: failure.daemon_initiated,
7359        approval: failure.approval.clone(),
7360    });
7361    Ok(())
7362}
7363
7364fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
7365    let _ = update_snapshot(snapshot, Some(module_id), |state| {
7366        state.health.last_action = Some(action);
7367        state.health.last_action_ms = Some(now_ms);
7368    });
7369}
7370
7371fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
7372    match status {
7373        HealthStatus::Ok => SupervisorHealthStatus::Ok,
7374        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
7375        HealthStatus::Failing => SupervisorHealthStatus::Failing,
7376    }
7377}
7378
7379/// Caps the metrics blob stored in the cached supervisor snapshot, which is
7380/// returned to every `supervisor.list` and `supervisor.health` caller.
7381///
7382/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
7383/// path: that request exists to return a module's complete metrics object, and
7384/// `ck health <module-id>` documents it as the way to see what the cached view
7385/// truncates. The asymmetry is the feature.
7386///
7387/// So a new caller must decide which side it is on rather than assume the cap is
7388/// universal. Reaching for it on a fresh-probe path would silently reintroduce
7389/// the truncation that path exists to avoid.
7390fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
7391    let metrics = metrics?;
7392    match serde_json::to_vec(&metrics) {
7393        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
7394            "truncated": true,
7395            "original_bytes": encoded.len(),
7396        })),
7397        Ok(_) | Err(_) => Some(metrics),
7398    }
7399}
7400
7401/// Spread health probes so a fleet-wide restart does not converge them.
7402///
7403/// The delay is derived from the module id and probe index rather than a random
7404/// source, so it is deterministic per module: a module keeps its own offset
7405/// across daemon restarts instead of re-rolling into a collision.
7406fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
7407    if cadence.is_zero() {
7408        return Duration::ZERO;
7409    }
7410    let cadence_ms = cadence.as_millis() as u64;
7411    // This early return is REDUNDANT, deliberately, and a mutation run will show
7412    // it surviving removal. Recording why here so the next person to notice does
7413    // not have to re-derive it:
7414    //
7415    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
7416    //   a zero cadence and builds the Duration from whole milliseconds, so a
7417    //   sub-millisecond cadence cannot come from config.
7418    // - Even if reached it changes no answer. The `.max(1)` below makes the span
7419    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
7420    //   -- exactly what this returns.
7421    //
7422    // Kept as a guard against a future widening of the config parser (accepting
7423    // microseconds, say), which would make the sub-millisecond case reachable.
7424    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
7425    // divides by zero. Remove this and nothing changes.
7426    if cadence_ms == 0 {
7427        return cadence;
7428    }
7429    // Note that this never returns less than one cadence, including for the FIRST
7430    // probe. So a freshly registered module reports health `unknown` for a full
7431    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
7432    // ready to answer.
7433    //
7434    // That is a property of the supervisor's schedule, not of any module: an
7435    // operator watching a restart sees `unknown` and cannot tell it from a module
7436    // that is slow to warm. Measured on two unrelated modules, both flipping to
7437    // `ok` between 22s and 32s after restart.
7438    //
7439    // Left as-is because spreading the first probe is what keeps a fleet-wide
7440    // restart from firing fourteen simultaneous probes into a cold machine. The
7441    // alternative -- probe at t+0 and jitter only from the second onward -- trades
7442    // that thundering herd for a faster first reading.
7443    let jitter_span = (cadence_ms / 10).max(1);
7444    let hash = module_id.as_bytes().iter().fold(
7445        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
7446        |acc, byte| {
7447            acc.wrapping_mul(1099511628211)
7448                .wrapping_add(u64::from(*byte))
7449        },
7450    );
7451    cadence + Duration::from_millis(hash % jitter_span)
7452}
7453
7454#[cfg(test)]
7455mod tests {
7456    use super::*;
7457
7458    #[test]
7459    fn registration_retirement_self_prefixed_owner_is_not_a_prefix_submodule() {
7460        let handle = SupervisorHandle::new();
7461        handle.set_spawn_nonce("prefix:owner", "owner-nonce".into());
7462        handle.set_reserved_prefixes("prefix:owner", &["prefix:".into()]);
7463        let owner = handle
7464            .reserved_hello_verification("prefix:owner", Some("owner-nonce"))
7465            .unwrap();
7466        assert!(owner.nonce_verified);
7467        assert!(
7468            owner.prefix_owner.is_none(),
7469            "an unreserved programmatic owner is not its own prefix child"
7470        );
7471        let child = handle
7472            .reserved_hello_verification("prefix:child", Some("owner-nonce"))
7473            .unwrap();
7474        assert!(child.nonce_verified);
7475        assert_eq!(child.prefix_owner.as_deref(), Some("prefix:owner"));
7476    }
7477
7478    #[test]
7479    fn readding_a_module_clears_its_rescan_removal_tombstone() {
7480        let handle = SupervisorHandle::new();
7481        let module_id = "readded-tombstone";
7482        handle.record_rescan_removal(module_id);
7483        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
7484
7485        handle.apply_identity_configuration(&ModuleSpec {
7486            script: None,
7487            module_id: module_id.to_string(),
7488            program: PathBuf::from("/test/module"),
7489            args: Vec::new(),
7490            env: Vec::new(),
7491            reserved: false,
7492            reserved_prefixes: Vec::new(),
7493            protocol: ModuleProtocol::Subc,
7494            overlap: Default::default(),
7495            carry_containment: false,
7496        });
7497
7498        assert!(
7499            handle.removal_tombstone_age_ms(module_id).is_none(),
7500            "a re-added module must not retain a stale removal tombstone"
7501        );
7502    }
7503
7504    /// What one module's owner looked like from the control plane at the
7505    /// instant after its first process was spawned.
7506    #[derive(Debug, PartialEq, Eq)]
7507    struct OwnerInSpawnWindow {
7508        module_id: String,
7509        configured: bool,
7510        on_roster: bool,
7511        admission_refusal: Option<&'static str>,
7512    }
7513
7514    /// A supervised module's process can connect, register, sync its scopes
7515    /// and describe them as soon as it is spawned. A plain `spawn` does not put
7516    /// the module on the roster until afterward; configured supervision does
7517    /// so before spawning. In both cases the owner must
7518    /// already read as configured, so a scoped `route.open` against it is
7519    /// refused as retryable `scope_not_synced` and not as terminal
7520    /// `scope_not_live` ("will never sync").
7521    ///
7522    /// The hook runs just after each initial spawn attempt, before returning to
7523    /// the caller, so no race with a real child's registration is needed.
7524    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7525    async fn a_new_module_is_configured_before_its_first_process_can_register() {
7526        use crate::scopes::ScopeTable;
7527        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
7528
7529        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
7530        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
7531            script: None,
7532            module_id: module_id.to_string(),
7533            program,
7534            args: Vec::new(),
7535            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7536                .into_iter()
7537                .map(|key| (key.to_string(), dir.path().display().to_string()))
7538                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7539                .collect(),
7540            reserved: false,
7541            reserved_prefixes: Vec::new(),
7542            protocol: ModuleProtocol::Subc,
7543            overlap: Default::default(),
7544            carry_containment: false,
7545        };
7546        let live = super::terminal_history_tests::fake_aft_stub_path();
7547        let missing = dir.path().join("definitely-missing-module");
7548
7549        let handle = SupervisorHandle::new();
7550        let mut supervisor =
7551            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7552                .with_handle(handle.clone());
7553        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
7554        let hook_handle = handle.clone();
7555        let hook_observed = Arc::clone(&observed);
7556        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7557            // Exactly what the control plane computes for a scoped route.open
7558            // naming this module as the owner of a scope it has not synced.
7559            let configured = hook_handle.is_configured(module_id);
7560            let selector = ScopeSelector::new(
7561                Principal::Reserved {
7562                    module_id: module_id.to_string(),
7563                },
7564                "s".to_string(),
7565                Some(1),
7566            );
7567            let carrier = Principal::Reserved {
7568                module_id: "carrier".to_string(),
7569            };
7570            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
7571                .admit(&carrier, module_id, &selector, configured)
7572            {
7573                Ok(_) => None,
7574                Err(refusal) => Some(refusal.code),
7575            };
7576            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
7577                module_id: module_id.to_string(),
7578                configured,
7579                on_roster: hook_handle.get(module_id).is_some(),
7580                admission_refusal,
7581            });
7582        })));
7583
7584        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
7585        let configured = supervisor
7586            .supervise_configured(stub("configured", live.clone()), true)
7587            .unwrap();
7588        let with_health = supervisor
7589            .supervise_configured_with_health(
7590                stub("with-health", live.clone()),
7591                true,
7592                HealthConfig::default(),
7593                None,
7594                RestartPolicy::default(),
7595            )
7596            .unwrap();
7597        // The failed-spawn path still puts the module on the roster (as
7598        // failed), so it is configured throughout.
7599        let failed = supervisor
7600            .supervise_configured_with_health(
7601                stub("failed-spawn", missing.clone()),
7602                true,
7603                HealthConfig::default(),
7604                None,
7605                RestartPolicy::default(),
7606            )
7607            .unwrap();
7608        // A failed plain `spawn` puts nothing on the roster, so its mark is
7609        // taken back once the spawn has failed.
7610        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
7611
7612        let expected = [
7613            "plain",
7614            "configured",
7615            "with-health",
7616            "failed-spawn",
7617            "spawn-error",
7618        ]
7619        .into_iter()
7620        .map(|module_id| OwnerInSpawnWindow {
7621            module_id: module_id.to_string(),
7622            configured: true,
7623            on_roster: !matches!(module_id, "plain" | "spawn-error"),
7624            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
7625        })
7626        .collect::<Vec<_>>();
7627        assert_eq!(*observed.lock().unwrap(), expected);
7628
7629        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
7630            assert!(
7631                handle.get(module_id).is_some(),
7632                "{module_id} is on the roster"
7633            );
7634            assert!(
7635                handle.is_configured(module_id),
7636                "{module_id} stays configured"
7637            );
7638        }
7639        assert!(handle.get("spawn-error").is_none());
7640        assert!(
7641            !handle.is_configured("spawn-error"),
7642            "a plain spawn that failed must not leave its module marked configured"
7643        );
7644
7645        // Leaving the roster clears the mark with it.
7646        handle.retire("failed-spawn");
7647        assert!(!handle.is_configured("failed-spawn"));
7648
7649        for module in [plain, configured, with_health] {
7650            module.stop().await.unwrap();
7651        }
7652        drop(failed);
7653    }
7654
7655    /// A configured module reads as starting until the supervisor has admitted
7656    /// its first process (exec confirmed, identity durably recorded), even
7657    /// though that process is already running by then and can register over
7658    /// HELLO, which is what lists it in catalog.list. So a client that needs
7659    /// the supervisor state must wait for that state; seeing the module in
7660    /// the catalog does not mean supervisor.list already says running.
7661    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7662    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
7663        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
7664        let spec = ModuleSpec {
7665            script: None,
7666            module_id: "spawned-not-admitted".to_string(),
7667            program: super::terminal_history_tests::fake_aft_stub_path(),
7668            args: Vec::new(),
7669            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7670                .into_iter()
7671                .map(|key| (key.to_string(), dir.path().display().to_string()))
7672                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7673                .collect(),
7674            reserved: false,
7675            reserved_prefixes: Vec::new(),
7676            protocol: ModuleProtocol::Subc,
7677            overlap: Default::default(),
7678            carry_containment: false,
7679        };
7680        let handle = SupervisorHandle::new();
7681        let mut supervisor =
7682            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7683                .with_handle(handle.clone());
7684        let observed = Arc::new(Mutex::new(None));
7685        let hook_handle = handle.clone();
7686        let hook_observed = Arc::clone(&observed);
7687        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7688            let state = hook_handle
7689                .get(module_id)
7690                .expect("a configured module is on the roster before it is spawned")
7691                .state()
7692                .unwrap();
7693            *hook_observed.lock().unwrap() = Some(state);
7694        })));
7695
7696        let module = supervisor.supervise_configured(spec, true).unwrap();
7697        assert_eq!(
7698            *observed.lock().unwrap(),
7699            Some(ModuleState::Starting),
7700            "between the spawn and its admission the supervisor still reports starting"
7701        );
7702        assert_eq!(module.state().unwrap(), ModuleState::Running);
7703        module.stop().await.unwrap();
7704    }
7705
7706    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
7707        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
7708        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
7709            snapshot.process_alive = true;
7710            snapshot.pid = Some(41);
7711            snapshot.spawned_at_ms = Some(42);
7712            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
7713            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
7714            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
7715                device: 43,
7716                inode: 44,
7717                #[cfg(windows)]
7718                windows_image: None,
7719            });
7720        })
7721        .unwrap();
7722        snapshot
7723    }
7724
7725    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
7726        let snapshot = lock_snapshot(snapshot).unwrap();
7727        assert!(!snapshot.process_alive);
7728        assert_eq!(snapshot.pid, None);
7729        assert_eq!(snapshot.spawned_at_ms, None);
7730        assert_eq!(snapshot.spawned_from, None);
7731        assert_eq!(snapshot.launch_spec_digest, None);
7732        assert_eq!(snapshot.spawned_file_identity, None);
7733    }
7734
7735    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7736    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
7737        let supervisor =
7738            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7739        let mut runtime = supervisor.runtime_config();
7740        runtime.test_seed_stale_facts_before_enable_spawn = true;
7741        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
7742        let mut child = None;
7743        let spec = ModuleSpec {
7744            script: None,
7745            module_id: "failed-enable-clears-facts".to_string(),
7746            program: PathBuf::from("/definitely/missing/failed-enable-module"),
7747            args: Vec::new(),
7748            env: Vec::new(),
7749            reserved: false,
7750            reserved_prefixes: Vec::new(),
7751            protocol: ModuleProtocol::Subc,
7752            overlap: Default::default(),
7753            carry_containment: false,
7754        };
7755
7756        let result = set_child_enabled(
7757            &spec,
7758            &runtime,
7759            &supervisor.registry,
7760            &supervisor.process_liveness,
7761            &snapshot,
7762            &mut child,
7763            true,
7764            &runtime.failure_context(false),
7765        )
7766        .await;
7767
7768        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
7769        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7770        assert_snapshot_process_facts_cleared(&snapshot);
7771    }
7772
7773    #[tokio::test]
7774    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
7775        let supervisor =
7776            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7777        let runtime = supervisor.runtime_config();
7778        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
7779            ModuleState::Restarting,
7780            true,
7781        )));
7782        let spec = ModuleSpec {
7783            script: None,
7784            module_id: "start-stranded-restarting".to_string(),
7785            program: super::terminal_history_tests::fake_aft_stub_path(),
7786            args: Vec::new(),
7787            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
7788            reserved: false,
7789            reserved_prefixes: Vec::new(),
7790            protocol: ModuleProtocol::None,
7791            overlap: Default::default(),
7792            carry_containment: false,
7793        };
7794        let mut child = None;
7795        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
7796        assert!(!super::set_child_enabled(
7797            &spec,
7798            &runtime,
7799            &Registry::default(),
7800            &supervisor.process_liveness,
7801            &snapshot,
7802            &mut child,
7803            true,
7804            &runtime.failure_context(false),
7805        )
7806        .await
7807        .unwrap());
7808        assert!(child.is_none());
7809        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
7810        assert!(super::set_child_enabled(
7811            &spec,
7812            &runtime,
7813            &Registry::default(),
7814            &supervisor.process_liveness,
7815            &snapshot,
7816            &mut child,
7817            true,
7818            &runtime.failure_context(false),
7819        )
7820        .await
7821        .unwrap());
7822        assert_eq!(
7823            lock_snapshot(&snapshot).unwrap().state,
7824            ModuleState::Running
7825        );
7826        let mut child = child.unwrap();
7827        child.start_kill().unwrap();
7828        child.wait().await.unwrap();
7829    }
7830
7831    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7832    async fn failed_reload_spawn_clears_current_process_facts() {
7833        let supervisor =
7834            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7835        let mut runtime = supervisor.runtime_config();
7836        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
7837        let snapshot = stale_process_snapshot(ModuleState::Running, true);
7838        let mut child = None;
7839        let spec = ModuleSpec {
7840            script: None,
7841            module_id: "failed-reload-clears-facts".to_string(),
7842            program: PathBuf::from("/unused/failed-reload-module"),
7843            args: Vec::new(),
7844            env: Vec::new(),
7845            reserved: false,
7846            reserved_prefixes: Vec::new(),
7847            protocol: ModuleProtocol::Subc,
7848            overlap: Default::default(),
7849            carry_containment: false,
7850        };
7851
7852        let result = handle_reload_spawn_failure(
7853            &spec,
7854            &runtime,
7855            &supervisor.process_liveness,
7856            &snapshot,
7857            &mut child,
7858            "forced reload spawn failure".to_string(),
7859        )
7860        .await;
7861
7862        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
7863        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7864        assert_snapshot_process_facts_cleared(&snapshot);
7865    }
7866
7867    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7868    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
7869        let supervisor =
7870            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7871        let snapshot = stale_process_snapshot(ModuleState::Running, true);
7872        let module = supervisor.supervised_module(
7873            ModuleSpec {
7874                script: None,
7875                module_id: "drop-clears-facts".to_string(),
7876                program: PathBuf::from("/unused/drop-module"),
7877                args: Vec::new(),
7878                env: Vec::new(),
7879                reserved: false,
7880                reserved_prefixes: Vec::new(),
7881                protocol: ModuleProtocol::Subc,
7882                overlap: Default::default(),
7883                carry_containment: false,
7884            },
7885            supervisor.runtime_config(),
7886            Arc::clone(&snapshot),
7887            None,
7888        );
7889        assert!(!module
7890            .inner
7891            .monitor
7892            .lock()
7893            .unwrap()
7894            .as_ref()
7895            .unwrap()
7896            .is_finished());
7897
7898        drop(module);
7899
7900        assert_eq!(
7901            lock_snapshot(&snapshot).unwrap().state,
7902            ModuleState::Stopped
7903        );
7904        assert_snapshot_process_facts_cleared(&snapshot);
7905    }
7906
7907    #[cfg(unix)]
7908    #[tokio::test]
7909    async fn rescan_preserves_running_protocol_until_respawn() {
7910        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
7911        let initial = ModuleSpec {
7912            script: None,
7913            module_id: "rescan-protocol".into(),
7914            program: PathBuf::from("/bin/sleep"),
7915            args: vec!["60".into()],
7916            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7917                .into_iter()
7918                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
7919                .collect(),
7920            reserved: false,
7921            reserved_prefixes: vec![],
7922            protocol: ModuleProtocol::None,
7923            overlap: Default::default(),
7924            carry_containment: false,
7925        };
7926        let supervisor =
7927            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7928        let module = supervisor.spawn(initial.clone()).unwrap();
7929        assert!(module.status().unwrap().live);
7930        let mut next = initial;
7931        next.protocol = ModuleProtocol::Subc;
7932        module
7933            .update_configuration(next.clone(), HealthConfig::default(), None)
7934            .await
7935            .unwrap();
7936        assert!(
7937            module.status().unwrap().live,
7938            "rescan must not require HELLO from the old non-wire process"
7939        );
7940        let runtime = supervisor.runtime_config();
7941        let action = on_child_exit(
7942            &next,
7943            RestartPolicy::default(),
7944            &supervisor.registry,
7945            &module.inner.snapshot,
7946            &runtime.terminal_ring,
7947            &runtime.spawn_events,
7948            &runtime.child_roster,
7949            ExitReport {
7950                kind: ExitKind::Clean,
7951                code: Some(0),
7952                signal: None,
7953                at_ms: unix_ms_now(),
7954            },
7955            true,
7956        )
7957        .await;
7958        assert!(
7959            matches!(action, NextAction::Restart { .. }),
7960            "the old non-wire process's clean exit must restart"
7961        );
7962        module.drain().await.unwrap();
7963    }
7964
7965    #[cfg(unix)]
7966    #[tokio::test]
7967    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
7968        use std::os::unix::fs::PermissionsExt;
7969        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
7970        let script = dir.join("module.sh");
7971        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
7972        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
7973        let record_path = dir.join("live-children.json");
7974        let supervisor =
7975            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7976                .with_live_children_record(&record_path);
7977        for (program, args) in [
7978            (PathBuf::from("sleep"), vec!["60".into()]),
7979            (script, vec![]),
7980        ] {
7981            let spec = ModuleSpec {
7982                script: None,
7983                module_id: "image-identity".into(),
7984                program: program.clone(),
7985                args,
7986                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7987                    .into_iter()
7988                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
7989                    .collect(),
7990                reserved: false,
7991                reserved_prefixes: vec![],
7992                protocol: ModuleProtocol::None,
7993                overlap: Default::default(),
7994                carry_containment: false,
7995            };
7996            let module = supervisor.spawn(spec).unwrap();
7997            #[cfg(target_os = "macos")]
7998            {
7999                // SETEXEC confirmation is asynchronous; the orphan record must
8000                // identify the final image, never the intermediate trampoline.
8001                let deadline = Instant::now() + Duration::from_secs(5);
8002                while crate::live_children::read_record(&record_path)
8003                    .unwrap()
8004                    .iter()
8005                    .all(|entry| entry.executable.is_none())
8006                {
8007                    assert!(Instant::now() < deadline, "module image was not confirmed");
8008                    tokio::time::sleep(Duration::from_millis(5)).await;
8009                }
8010            }
8011            let entry = crate::live_children::read_record(&record_path)
8012                .unwrap()
8013                .pop()
8014                .unwrap();
8015            let observed = subc_os::Process::open(entry.pid)
8016                .unwrap()
8017                .unwrap()
8018                .observe()
8019                .unwrap();
8020            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
8021            module.drain().await.unwrap();
8022            assert_eq!(
8023                verdict,
8024                crate::live_children::IdentityVerdict::Matches,
8025                "program {program:?}: recorded {entry:?}, observed {observed:?}"
8026            );
8027        }
8028    }
8029
8030    #[cfg(unix)]
8031    fn http_fixture(
8032        dir: &std::path::Path,
8033        url: &str,
8034        threshold: u32,
8035    ) -> crate::daemon_config::ConfiguredModule {
8036        let path = dir.join("subc.jsonc");
8037        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
8038            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
8039            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
8040            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
8041            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
8042        }}}).to_string()).unwrap();
8043        crate::daemon_config::load(&path)
8044            .unwrap()
8045            .unwrap()
8046            .modules
8047            .pop()
8048            .unwrap()
8049    }
8050
8051    #[cfg(unix)]
8052    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
8053        timeout(Duration::from_secs(5), async {
8054            loop {
8055                if module.status().unwrap().health.status == status {
8056                    break;
8057                }
8058                sleep(Duration::from_millis(5)).await;
8059            }
8060        })
8061        .await
8062        .unwrap_or_else(|_| {
8063            panic!(
8064                "expected {status:?}, got {:?}",
8065                module.status().unwrap().health
8066            )
8067        });
8068    }
8069
8070    #[cfg(unix)]
8071    #[tokio::test]
8072    async fn http_health_status_flips_ok_failing_ok() {
8073        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8074        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
8075        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8076        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
8077        let serving_status = status.clone();
8078        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8079        let server = tokio::spawn(async move {
8080            loop {
8081                let (mut stream, _) = listener.accept().await.unwrap();
8082                let mut request = [0u8; 2048];
8083                let count = stream.read(&mut request).await.unwrap();
8084                assert!(count > 0, "a probe must send an HTTP request");
8085                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
8086                let body = if code == 200 {
8087                    "ready"
8088                } else {
8089                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
8090                };
8091                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
8092                let _ = stream.write_all(response.as_bytes()).await;
8093            }
8094        });
8095        let configured = http_fixture(&dir, &url, 1000);
8096        let module =
8097            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8098                .supervise_configured_with_health(
8099                    configured.module_spec(),
8100                    true,
8101                    configured.health,
8102                    configured.drain_timeout_ms,
8103                    configured.restart,
8104                )
8105                .unwrap();
8106        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8107        status.store(503, std::sync::atomic::Ordering::SeqCst);
8108        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
8109        assert!(module
8110            .status()
8111            .unwrap()
8112            .health
8113            .detail
8114            .unwrap()
8115            .contains("scratch failure"));
8116        status.store(200, std::sync::atomic::Ordering::SeqCst);
8117        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8118        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
8119        let before = module.status().unwrap();
8120        let (spec, mut health) = module.configuration().unwrap();
8121        health.http = None;
8122        module
8123            .update_configuration(spec.clone(), health.clone(), Some(10))
8124            .await
8125            .unwrap();
8126        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
8127        health.http = Some(url);
8128        module
8129            .update_configuration(spec, health, Some(10))
8130            .await
8131            .unwrap();
8132        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8133        assert_eq!(
8134            module.status().unwrap().pid,
8135            before.pid,
8136            "changing a probe must apply live, not restart its process"
8137        );
8138        let (spec, mut health) = module.configuration().unwrap();
8139        health.failure_threshold = 2;
8140        module
8141            .update_configuration(spec, health, Some(10))
8142            .await
8143            .unwrap();
8144        status.store(503, std::sync::atomic::Ordering::SeqCst);
8145        timeout(Duration::from_secs(5), async {
8146            while module.status().unwrap().spawn_generation == before.spawn_generation {
8147                sleep(Duration::from_millis(5)).await;
8148            }
8149        })
8150        .await
8151        .expect("sustained HTTP 503 responses must trigger the health restart policy");
8152        module.drain().await.unwrap();
8153        server.abort();
8154    }
8155
8156    #[cfg(unix)]
8157    #[tokio::test]
8158    async fn http_health_replacement_recovers_after_one_restart() {
8159        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8160        let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
8161        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8162        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8163        let mut configured = http_fixture(&dir, &url, 2);
8164        configured.restart.max_restarts = 2;
8165        let module =
8166            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8167                .supervise_configured_with_health(
8168                    configured.module_spec(),
8169                    true,
8170                    configured.health,
8171                    configured.drain_timeout_ms,
8172                    configured.restart,
8173                )
8174                .unwrap();
8175        let first_generation = module.status().unwrap().spawn_generation;
8176        let observed = module.clone();
8177        let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
8178        let server = tokio::spawn(async move {
8179            let mut generation = first_generation;
8180            let mut probes = 0;
8181            loop {
8182                let (mut stream, _) = listener.accept().await.unwrap();
8183                let mut request = [0; 2048];
8184                assert!(stream.read(&mut request).await.unwrap() > 0);
8185                let current = observed.status().unwrap().spawn_generation;
8186                if current != generation {
8187                    generation = current;
8188                    probes = 0;
8189                }
8190                probes += 1;
8191                // The original process never recovers. Each replacement has one
8192                // transient miss, then answers successfully for its remaining life.
8193                let healthy = generation != first_generation && probes > 1;
8194                let code = if healthy { 200 } else { 503 };
8195                let response = format!(
8196                    "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
8197                );
8198                let _ = stream.write_all(response.as_bytes()).await;
8199                if healthy {
8200                    healthy_tx.send(()).await.unwrap();
8201                }
8202            }
8203        });
8204        timeout(Duration::from_secs(5), async {
8205            let mut healthy_probes = 0;
8206            loop {
8207                if module.state().unwrap() == ModuleState::Failed {
8208                    break;
8209                }
8210                while healthy_rx.try_recv().is_ok() {
8211                    healthy_probes += 1;
8212                }
8213                if healthy_probes >= 3
8214                    && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
8215                {
8216                    break;
8217                }
8218                sleep(Duration::from_millis(5)).await;
8219            }
8220        })
8221        .await
8222        .expect("the replacement must settle healthy without exhausting its restart budget");
8223        let status = module.status().unwrap();
8224        module.drain().await.unwrap();
8225        server.abort();
8226        assert_eq!(status.state, ModuleState::Running);
8227        assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
8228        assert_eq!(status.health.consecutive_failures, 0);
8229        assert_eq!(status.lifetime_restarts, 1);
8230        assert_eq!(status.restart_count, 1);
8231    }
8232
8233    #[cfg(unix)]
8234    #[tokio::test]
8235    async fn http_health_sustained_failure_exhausts_restart_budget() {
8236        use std::collections::BTreeMap;
8237        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8238        let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
8239        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8240        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8241        let mut configured = http_fixture(&dir, &url, 2);
8242        configured.restart.max_restarts = 2;
8243        let module =
8244            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8245                .supervise_configured_with_health(
8246                    configured.module_spec(),
8247                    true,
8248                    configured.health,
8249                    configured.drain_timeout_ms,
8250                    configured.restart,
8251                )
8252                .unwrap();
8253        let observed = module.clone();
8254        let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
8255        let served = probes.clone();
8256        let server = tokio::spawn(async move {
8257            loop {
8258                let (mut stream, _) = listener.accept().await.unwrap();
8259                let mut request = [0; 2048];
8260                assert!(stream.read(&mut request).await.unwrap() > 0);
8261                *served
8262                    .lock()
8263                    .unwrap()
8264                    .entry(observed.status().unwrap().spawn_generation)
8265                    .or_default() += 1;
8266                let _ = stream
8267                    .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
8268                    .await;
8269            }
8270        });
8271        timeout(Duration::from_secs(5), async {
8272            while module.state().unwrap() != ModuleState::Failed {
8273                sleep(Duration::from_millis(5)).await;
8274            }
8275        })
8276        .await
8277        .expect("sustained failures must still exhaust the health restart budget");
8278        let status = module.status().unwrap();
8279        server.abort();
8280        assert_eq!(status.restart_count, 2);
8281        assert_eq!(status.lifetime_restarts, 2);
8282        assert!(!status.process_alive);
8283        assert_eq!(
8284            probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
8285            vec![2, 2, 2]
8286        );
8287    }
8288
8289    #[cfg(unix)]
8290    #[tokio::test]
8291    async fn spawn_clears_the_previous_process_health() {
8292        let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
8293        let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
8294        configured.health.cadence = Duration::from_secs(3600);
8295        let module =
8296            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8297                .supervise_configured_with_health(
8298                    configured.module_spec(),
8299                    false,
8300                    configured.health,
8301                    configured.drain_timeout_ms,
8302                    configured.restart,
8303                )
8304                .unwrap();
8305        update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
8306            state.health.status = SupervisorHealthStatus::Unresponsive;
8307            state.health.consecutive_failures = 3;
8308            state.health.detail = Some("old process refused connections".into());
8309            state.health.metrics = Some(serde_json::json!({"old": true}));
8310            state.health.last_probe_ms = Some(123);
8311            state.health.last_action = Some("restart".into());
8312            state.health.last_action_ms = Some(124);
8313        })
8314        .unwrap();
8315        module.set_enabled(true).await.unwrap();
8316        let health = module.status().unwrap().health;
8317        module.drain().await.unwrap();
8318        assert_eq!(health.consecutive_failures, 0);
8319        assert_eq!(health.status, SupervisorHealthStatus::Unknown);
8320        assert_eq!(health.detail, None);
8321        assert_eq!(health.metrics, None);
8322        assert_eq!(health.last_probe_ms, None);
8323        assert_eq!(health.last_action.as_deref(), Some("restart"));
8324        assert_eq!(health.last_action_ms, Some(124));
8325    }
8326
8327    #[cfg(unix)]
8328    #[tokio::test]
8329    async fn http_health_no_listener_restarts_after_consecutive_failures() {
8330        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
8331        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8332        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
8333        drop(unused);
8334        let configured = http_fixture(&dir, &url, 2);
8335        let module =
8336            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8337                .supervise_configured_with_health(
8338                    configured.module_spec(),
8339                    true,
8340                    configured.health,
8341                    configured.drain_timeout_ms,
8342                    configured.restart,
8343                )
8344                .unwrap();
8345        let before = module.status().unwrap().spawn_generation;
8346        timeout(Duration::from_secs(5), async {
8347            loop {
8348                let status = module.status().unwrap();
8349                if status.spawn_generation > before {
8350                    assert!(status.lifetime_restarts > 0);
8351                    break;
8352                }
8353                sleep(Duration::from_millis(5)).await;
8354            }
8355        })
8356        .await
8357        .expect("sustained HTTP refusal must trigger the health restart policy");
8358        module.drain().await.unwrap();
8359    }
8360
8361    #[cfg(unix)]
8362    #[tokio::test]
8363    async fn http_health_timeout_honours_deadline() {
8364        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8365        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8366        let server = tokio::spawn(async move {
8367            let _held = listener.accept().await.unwrap();
8368            std::future::pending::<()>().await;
8369        });
8370        let error = timeout(
8371            Duration::from_secs(1),
8372            probe_http_health(&url, Duration::from_millis(10)),
8373        )
8374        .await
8375        .expect("the probe must enforce its own deadline")
8376        .unwrap_err();
8377        server.abort();
8378        assert!(error.to_string().contains("timed out"));
8379    }
8380
8381    #[cfg(unix)]
8382    #[tokio::test]
8383    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
8384        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8385        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
8386        let url = format!(
8387            "http://localhost:{}/healthz",
8388            listener.local_addr().unwrap().port()
8389        );
8390        let server = tokio::spawn(async move {
8391            let (mut stream, _) = listener.accept().await.unwrap();
8392            let mut request = [0u8; 2048];
8393            assert!(stream.read(&mut request).await.unwrap() > 0);
8394            stream
8395                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
8396                .await
8397                .unwrap();
8398        });
8399        // The deadline only bounds a hang. A probe that never tried the IPv6
8400        // address would be refused on 127.0.0.1 and fail at once, so a longer
8401        // deadline does not weaken the assertion; one second timed out under a
8402        // loaded parallel test run.
8403        assert_eq!(
8404            probe_http_health(&url, Duration::from_secs(10))
8405                .await
8406                .unwrap()
8407                .status,
8408            HealthStatus::Ok
8409        );
8410        server.await.unwrap();
8411    }
8412
8413    #[cfg(unix)]
8414    #[tokio::test]
8415    async fn http_health_timeout_keeps_partial_status_and_body() {
8416        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8417        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8418        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8419        let server = tokio::spawn(async move {
8420            let (mut stream, _) = listener.accept().await.unwrap();
8421            let mut request = [0u8; 2048];
8422            assert!(stream.read(&mut request).await.unwrap() > 0);
8423            stream
8424                .write_all(
8425                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
8426                )
8427                .await
8428                .unwrap();
8429            std::future::pending::<()>().await;
8430        });
8431        let error = probe_http_health(&url, Duration::from_secs(1))
8432            .await
8433            .unwrap_err()
8434            .to_string();
8435        server.abort();
8436        assert!(
8437            error.contains("timed out")
8438                && error.contains("503 Unavailable")
8439                && error.contains("partial diagnostic"),
8440            "{error}"
8441        );
8442    }
8443
8444    #[cfg(unix)]
8445    #[tokio::test]
8446    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
8447        use tokio::io::{AsyncReadExt, AsyncWriteExt};
8448        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8449        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8450        let server = tokio::spawn(async move {
8451            let (mut stream, _) = listener.accept().await.unwrap();
8452            let mut request = [0u8; 2048];
8453            assert!(stream.read(&mut request).await.unwrap() > 0);
8454            let body = format!("{}not-in-diagnostic", "x".repeat(400));
8455            let response = format!(
8456                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
8457                body.len()
8458            );
8459            stream.write_all(response.as_bytes()).await.unwrap();
8460        });
8461        let error = probe_http_health(&url, Duration::from_secs(1))
8462            .await
8463            .unwrap_err()
8464            .to_string();
8465        server.await.unwrap();
8466        assert!(error.ends_with(&"x".repeat(200)), "{error}");
8467        assert_eq!(error.split(": ").last().unwrap().len(), 200);
8468        assert!(!error.contains("not-in-diagnostic"));
8469    }
8470
8471    #[cfg(unix)]
8472    #[tokio::test]
8473    async fn http_health_real_nats_server_monitoring() {
8474        if std::process::Command::new("nats-server")
8475            .arg("--version")
8476            .env("XDG_DATA_HOME", std::env::temp_dir())
8477            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
8478            .env("XDG_CONFIG_HOME", std::env::temp_dir())
8479            .output()
8480            .is_err()
8481        {
8482            eprintln!(
8483                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
8484            );
8485            return;
8486        }
8487        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
8488        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8489        let port = monitor.local_addr().unwrap().port();
8490        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8491        let client_port = client.local_addr().unwrap().port();
8492        let config = dir.join("server.conf");
8493        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
8494        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
8495        configured.program = PathBuf::from("nats-server");
8496        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
8497        drop(monitor);
8498        drop(client);
8499        let module =
8500            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8501                .supervise_configured_with_health(
8502                    configured.module_spec(),
8503                    true,
8504                    configured.health,
8505                    configured.drain_timeout_ms,
8506                    configured.restart,
8507                )
8508                .unwrap();
8509        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8510        module.drain().await.unwrap();
8511    }
8512
8513    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8514    async fn configuration_update_does_not_replace_captured_running_process_facts() {
8515        let supervisor =
8516            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8517        let snapshot = stale_process_snapshot(ModuleState::Running, true);
8518        let initial = ModuleSpec {
8519            script: None,
8520            module_id: "rescan-preserves-spawn-facts".to_string(),
8521            program: PathBuf::from("/spawned/module"),
8522            args: Vec::new(),
8523            env: Vec::new(),
8524            reserved: false,
8525            reserved_prefixes: Vec::new(),
8526            protocol: ModuleProtocol::Subc,
8527            overlap: Default::default(),
8528            carry_containment: false,
8529        };
8530        let module = supervisor.supervised_module(
8531            initial.clone(),
8532            supervisor.runtime_config(),
8533            snapshot,
8534            None,
8535        );
8536        let before = module.status().unwrap();
8537        let mut replacement = initial;
8538        replacement.program = PathBuf::from("/rescanned/replacement-module");
8539
8540        module
8541            .update_configuration(replacement, HealthConfig::default(), None)
8542            .await
8543            .unwrap();
8544
8545        let after = module.status().unwrap();
8546        assert_eq!(after.pid, before.pid);
8547        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
8548        assert_eq!(after.spawned_from, before.spawned_from);
8549        drop(module);
8550    }
8551}
8552
8553fn unix_ms_now() -> u64 {
8554    SystemTime::now()
8555        .duration_since(UNIX_EPOCH)
8556        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
8557        .unwrap_or(0)
8558}
8559
8560async fn supervise_loop(
8561    mut spec: ModuleSpec,
8562    mut runtime: SupervisorRuntimeConfig,
8563    registry: Arc<Registry>,
8564    process_liveness: Arc<SupervisorProcessLiveness>,
8565    snapshot: SharedSnapshot,
8566    mut child: Option<SupervisedChild>,
8567    mut commands: mpsc::Receiver<SupervisorCommand>,
8568) {
8569    let mut health_probe = HealthProbeRuntime::default();
8570    // Registry writes (including embedded callers) notify this module only.
8571    // Subscribe before the first refresh; watch retains changes that arrive
8572    // while commands or probes are running. No polling fallback is needed.
8573    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
8574        Ok(changes) => Some(changes),
8575        Err(err) => {
8576            // A poisoned registry cannot accept further writes, so it cannot
8577            // recover via a timer. Keep exit and command handling alive.
8578            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
8579            None
8580        }
8581    };
8582    // All restart backoffs run here, including health and operator requests.
8583    // While one is pending the loop serves commands, so disable or drain can
8584    // cancel the replacement without spawning a process just to stop it.
8585    let mut pending_respawn: Option<PendingRespawn> = None;
8586    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
8587    // before anything else so a stop that interrupted a swap runs at once.
8588    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
8589    let mut previous_instance_recheck = tokio::time::interval_at(
8590        Instant::now() + Duration::from_secs(5),
8591        Duration::from_secs(5),
8592    );
8593    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
8594    'supervision: loop {
8595        #[cfg(test)]
8596        {
8597            lock_snapshot(&snapshot).unwrap().actor_select = None;
8598        }
8599        #[cfg(target_os = "macos")]
8600        if let Some(active) = child.as_mut() {
8601            active.confirm_privacy_exec().await;
8602        }
8603        if let Some(scheduled) = runtime
8604            .scheduled_respawn
8605            .lock()
8606            .unwrap_or_else(|p| p.into_inner())
8607            .take()
8608        {
8609            runtime.recovery_is_daemon = scheduled.daemon_initiated;
8610            pending_respawn = Some(scheduled);
8611        }
8612        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
8613            pending_respawn = None;
8614            cancel_deferred_reload(
8615                &runtime,
8616                &spec.module_id,
8617                "respawn cancelled by a supervisor command",
8618            );
8619        }
8620        if child.is_none()
8621            && pending_respawn.is_none()
8622            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
8623        {
8624            cancel_deferred_reload(
8625                &runtime,
8626                &spec.module_id,
8627                "respawn cancelled before a replacement was spawned",
8628            );
8629            let mut stranded = false;
8630            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8631                state.respawn_pending = false;
8632                state.coalesced_restart_pending = false;
8633                if matches!(
8634                    state.state,
8635                    ModuleState::Restarting
8636                        | ModuleState::Starting
8637                        | ModuleState::Draining
8638                        | ModuleState::Unresponsive
8639                ) {
8640                    error!(module_id = %spec.module_id, state = ?state.state,
8641                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
8642                    stranded = true;
8643                }
8644            });
8645            if stranded {
8646                fail_snapshot(
8647                    &snapshot,
8648                    Some(&spec.module_id),
8649                    None,
8650                    &runtime.failure_context(runtime.recovery_is_daemon),
8651                );
8652            }
8653        }
8654        if child.is_none()
8655            && (runtime.child_roster.is_closed()
8656                || pending_respawn.is_none()
8657                || !lock_snapshot(&snapshot)
8658                    .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
8659        {
8660            containment::terminate(&runtime.child_roster.containment).await;
8661        }
8662        if let Some(command) = requeued.pop_front() {
8663            if !handle_supervisor_command(
8664                command,
8665                &mut spec,
8666                &mut runtime,
8667                &registry,
8668                &process_liveness,
8669                &snapshot,
8670                &mut child,
8671                &mut commands,
8672                &mut requeued,
8673            )
8674            .await
8675            {
8676                containment::terminate(&runtime.child_roster.containment).await;
8677                return;
8678            }
8679            if child.is_some() || !respawn_still_pending(&snapshot) {
8680                pending_respawn = None;
8681                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8682                    state.respawn_pending = false
8683                });
8684            }
8685            continue;
8686        }
8687        if child.is_some() {
8688            runtime.recovery_is_daemon = true;
8689            #[cfg(test)]
8690            {
8691                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
8692            }
8693            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
8694            let active = child.as_mut().expect("child checked above");
8695            if active.registered_since_spawn() {
8696                if let Some(delivery) = &runtime.config_delivery {
8697                    delivery.queue_registered(
8698                        &spec.module_id,
8699                        runtime.forwarding.as_ref(),
8700                        &registry,
8701                        active,
8702                    );
8703                }
8704            }
8705            if runtime.child_roster.is_closed() || !active.awaiting_registration() {
8706                active.registration_deadline = None;
8707            }
8708            let registration_deadline = active.registration_deadline;
8709            let no_checks = runtime.health.http.is_none()
8710                && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
8711                    || (health_probe.registered_connection.is_some() && !health_probe.advertised));
8712            let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
8713                state
8714                    .failed_retry
8715                    .uptime_reset_at(no_checks, runtime.restart_policy.window)
8716            });
8717            let wake_at = match (health_probe.wake_at(), uptime_reset) {
8718                (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
8719                (probe, None) => probe,
8720            };
8721            let wake_at = match (wake_at, registration_deadline) {
8722                (Some(wake), Some(register)) => Some(wake.min(register)),
8723                (wake, register) => wake.or(register),
8724            };
8725            #[cfg(test)]
8726            let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
8727            let probe_sleep = supervisor_sleep(wake_at);
8728            tokio::pin!(probe_sleep);
8729            // A `protocol: "none"` child (a plain process that never registers
8730            // over the subc wire, such as nats-server) with no HTTP health
8731            // check has nothing to probe, so it waits only for its exit or a
8732            // supervisor command. A rescan that adds an HTTP check is a command.
8733            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
8734            #[cfg(test)]
8735            {
8736                let registration_pending = wire_child
8737                    && registration_changes
8738                        .as_ref()
8739                        .is_some_and(|changes| changes.has_changed().unwrap());
8740                let mut state = lock_snapshot(&snapshot).unwrap();
8741                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
8742                    generation: state.spawn_generation,
8743                    turn: state.actor_turns,
8744                    registered_connection: health_probe.registered_connection,
8745                    next_probe_at: health_probe.next_probe_at,
8746                    wake_after,
8747                });
8748            }
8749            let registration = active
8750                .awaiting_registration()
8751                .then(|| active.registration_spawn.clone())
8752                .flatten();
8753            #[cfg(all(test, target_os = "linux"))]
8754            let registration_before_exit = !runtime.child_roster.admission_event_for_test(
8755                &spec.module_id,
8756                "observe_registration_before_exit",
8757                Some(active.pid),
8758            ) && registration.is_some();
8759            #[cfg(not(all(test, target_os = "linux")))]
8760            let registration_before_exit = false;
8761            #[cfg(all(test, target_os = "linux"))]
8762            let registration_child_is_running = |child: &mut SupervisedChild| {
8763                // Model the interval after SIGKILL is accepted but before the
8764                // child becomes waitable, without delaying the actual kill.
8765                registration_before_exit || registration_child_is_running(child)
8766            };
8767            let registration_failed = async {
8768                match &registration {
8769                    Some(spawn) => spawn.wait_registration_failure().await,
8770                    None => std::future::pending().await,
8771                }
8772            };
8773            let active_child = child.as_mut().expect("child checked above");
8774            tokio::select! {
8775                _ = registration_failed => {
8776                    if !active_child.registered_since_spawn() && registration_child_is_running(active_child) {
8777                        active_child.registration_deadline = None;
8778                        active_child.spawn_failure = Some("registration_channel_closed".into());
8779                        if let Err(error) = stop_unregistered_child(active_child).await {
8780                            fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8781                                &runtime.spawn_events, format!("registration_channel_closed: {error}"),
8782                                &runtime.failure_context(true));
8783                            child = None;
8784                        }
8785                    }
8786                    continue;
8787                }
8788                wait_result = active_child.wait(), if !registration_before_exit => {
8789                    if let Ok(mut state) = lock_snapshot(&snapshot) {
8790                        state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
8791                    }
8792                    #[cfg(test)]
8793                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8794                    // Every arm below that gives up on the CHILD must keep the
8795                    // supervision task itself alive (child = None, loop
8796                    // continues into command-serving mode). Returning here
8797                    // closes the command channel, which makes the module
8798                    // permanently unrestartable in-band: a clean child exit
8799                    // of an enabled module once wedged the fleet this way
8800                    // ('supervisor command channel is closed') and required a
8801                    // full daemon restart to recover.
8802                    let exit_report = match wait_result {
8803                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
8804                        Err(err) => {
8805                            active_child.drain_stderr(&spec.module_id).await;
8806                            fail_after_wait_error(
8807                                &spec.module_id,
8808                                &snapshot,
8809                                &runtime.terminal_ring,
8810                                &runtime.spawn_events,
8811                                format!("failed to wait for supervised module: {err}"),
8812                            &runtime.failure_context(true),
8813                            );
8814                            untrack_if_registration_released(
8815                                &process_liveness,
8816                                &registry,
8817                                &spec.module_id,
8818                                &snapshot,
8819                            );
8820                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
8821                            child = None;
8822                            continue;
8823                        }
8824                    };
8825                    active_child.drain_stderr(&spec.module_id).await;
8826
8827                    let delivery=runtime.config_delivery.clone();
8828                    let exited_spawn=active_child.spawned_at;
8829                    let spawn=active_child.config_delivery.clone();
8830                    let refusal=active_child.spawn_failure.as_deref().filter(|detail|detail.starts_with("program_refused:") && !active_child.live_program_binding.as_ref().is_some_and(|binding|binding.is_refused())).zip(active_child.admission_base.clone()).map(|(detail,base)|(base,subc_config::raw_digest(detail.as_bytes())));
8831                    let module_id=spec.module_id.clone();
8832                    // Publication may own the store mutex for local durable I/O.
8833                    // Keep commands selectable until the child's exit is recorded;
8834                    // no respawn disposition is chosen before that observation.
8835                    let observed=async move {
8836                        if let Some(delivery)=delivery {
8837                            delivery.0.on_blocking(move |owner| {
8838                                if let Some(spawn)=spawn { owner.delivery_child_exited(&spawn)?; }
8839                                if let Some((base,identity))=refusal {owner.record_program_refusal(&module_id,&base,identity)?;}
8840                                Ok(())
8841                            }).await
8842                        } else {Ok(())}
8843                    };
8844                    tokio::pin!(observed);
8845                    loop {
8846                        tokio::select! {
8847                            result=&mut observed=>{
8848                                if let Err(error)=result {warn!(module_id=%spec.module_id,%error,"configuration exit observation failed");}
8849                                break;
8850                            }
8851                            command=commands.recv()=>{
8852                                let Some(command)=command else {containment::terminate(&runtime.child_roster.containment).await;return;};
8853                                let deferred=||SuperviseError::ConfigDeliveryDeferred {module_id:spec.module_id.clone()};
8854                                let command=match command {
8855                                    SupervisorCommand::Restart {reply,..}|SupervisorCommand::Reload {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8856                                    SupervisorCommand::Swap {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8857                                    command @ (SupervisorCommand::ConfigApplyRestart {..}|SupervisorCommand::ConfigPolicyProbe {..}|SupervisorCommand::ConfigDeliveryFailed {..})=>{requeued.push_back(command);continue;}
8858                                    command=>command,
8859                                };
8860                                if !handle_supervisor_command(command,&mut spec,&mut runtime,&registry,&process_liveness,&snapshot,&mut child,&mut commands,&mut requeued).await {
8861                                    containment::terminate(&runtime.child_roster.containment).await;return;
8862                                }
8863                                if child.as_ref().is_none_or(|current|current.spawned_at!=exited_spawn) {continue 'supervision;}
8864                            }
8865                        }
8866                    }
8867                    let active_child=child.as_mut().expect("exit observation retained its child");
8868                    // An unavailable image can race the program's own exit.
8869                    // Only the admission kill is budget-free; another exit must
8870                    // retain its ordinary crash/clean-exit accounting.
8871                    let admission_kill = admission_exit::is_admission_kill(active_child.admission_kill_sent, &exit_report);
8872                    let deferred_spawn=admission_kill && active_child.spawn_failure.as_deref().is_some_and(|detail|detail.starts_with("config_delivery_deferred:"));
8873                    let refused_program = active_child.live_program_binding.as_ref().is_some_and(|binding| binding.is_refused());
8874                    let exit_held = !deferred_spawn && (refused_program || runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.exit_is_held(&spec.module_id)));
8875                    let parked = exit_held && (!refused_program || active_child.admission_base.is_some())
8876                        && try_park_config_attempt(&spec, &runtime, &snapshot, &runtime.failure_context(true),
8877                            if refused_program { active_child.admission_base.as_ref() } else { None })
8878                            .unwrap_or_else(|error| { warn!(module_id=%spec.module_id, %error, "exit hold could not be recorded; using ordinary exit accounting"); false });
8879                    let next = if deferred_spawn {
8880                        record_terminal_with_detail(&spec.module_id,&runtime.terminal_ring,&runtime.spawn_events,&exit_report,TerminalDisposition::Restarting,active_child.spawn_failure.clone());
8881                        let _=update_snapshot(&snapshot,Some(&spec.module_id),|state| {
8882                            state.state=ModuleState::Restarting;
8883                            clear_current_process_facts(state);
8884                            state.last_exit=Some(exit_report.clone());
8885                        });
8886                        NextAction::Restart {schedule:None}
8887                    } else if parked {
8888                        record_terminal_with_detail(&spec.module_id, &runtime.terminal_ring, &runtime.spawn_events, &exit_report, TerminalDisposition::Stopped, active_child.spawn_failure.clone().or_else(|| Some("config_gated".into())));
8889                        NextAction::Stop { registration_released: registry.get_module(&spec.module_id).ok().flatten().is_none() }
8890                    } else { on_child_exit(
8891                        &spec,
8892                        runtime.restart_policy,
8893                        &registry,
8894                        &snapshot,
8895                        &runtime.terminal_ring,
8896                        &runtime.spawn_events,
8897                        &runtime.child_roster,
8898                        exit_report,
8899                        true,
8900                    ).await };
8901                    runtime.start_requester = None;
8902                    runtime.pending_activity_restart = false;
8903                    // The exit is recorded, so a daemon shutdown may stop
8904                    // waiting for this child (see `SupervisedChild::wait`).
8905                    let carry = matches!(next, NextAction::Restart { .. })
8906                        && active_child.spawn_failure.is_none()
8907                        && !runtime.child_roster.is_closed();
8908                    if let Err(error) = active_child.finish_exit(carry).await {
8909                        containment::terminate(&runtime.child_roster.containment).await;
8910                        fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8911                            &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
8912                            &runtime.failure_context(true));
8913                        child = None;
8914                        continue;
8915                    }
8916                    match next {
8917                        NextAction::Stop { registration_released } => {
8918                            if registration_released {
8919                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
8920                            }
8921                            child = None;
8922                        }
8923                        NextAction::Restart { schedule } => {
8924                            let delay = if deferred_spawn {CONFIG_ADMISSION_RETRY_DELAY} else {schedule.map_or(
8925                                runtime.restart_policy.delay_for_restart(0),
8926                                |schedule| schedule.delay,
8927                            )};
8928                            if let Some(schedule) = schedule {
8929                                log_crash_respawn(&spec.module_id, schedule);
8930                            }
8931                            // The exited child is fully recorded at this point,
8932                            // so release it and count the backoff down in the
8933                            // command-serving branch below rather than sleeping
8934                            // here: commands cannot be received from inside this
8935                            // select arm, and an operator disable or drain that
8936                            // arrives during the backoff must cancel the pending
8937                            // respawn instead of waiting for it to spawn first.
8938                            child = None;
8939                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true, approval: None });
8940                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
8941                        }
8942                    }
8943                }
8944                command = commands.recv() => {
8945                    #[cfg(test)]
8946                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8947                    let Some(command) = command else {
8948                        containment::terminate(&runtime.child_roster.containment).await;
8949                        return;
8950                    };
8951                    if !handle_supervisor_command(
8952                        command,
8953                        &mut spec,
8954                        &mut runtime,
8955                        &registry,
8956                        &process_liveness,
8957                        &snapshot,
8958                        &mut child,
8959                        &mut commands,
8960                        &mut requeued,
8961                    ).await {
8962                        containment::terminate(&runtime.child_roster.containment).await;
8963                        return;
8964                    }
8965                }
8966                _ = async {
8967                    match registration_changes.as_mut() {
8968                        Some(changes) => { let _ = changes.changed().await; }
8969                        None => std::future::pending().await,
8970                    }
8971                }, if wire_child => {
8972                    #[cfg(test)]
8973                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8974                }
8975                _ = &mut probe_sleep => {
8976                    #[cfg(test)]
8977                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8978                    if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
8979                        // Recheck after the wake: HELLO, exit or shutdown may
8980                        // have become ready in the same executor tick.
8981                        active_child.registration_deadline = None;
8982                        if !runtime.child_roster.is_closed()
8983                            && !active_child.registered_since_spawn()
8984                            && registration_child_is_running(active_child)
8985                        {
8986                            warn!(module_id = %spec.module_id, pid = active_child.pid,
8987                                wait_secs = runtime.register_timeout.as_secs(),
8988                                "supervised module registration_timeout: first HELLO deadline expired; stopping child");
8989                            active_child.spawn_failure = Some("registration_timeout".into());
8990                            if let Err(error) = stop_unregistered_child(active_child).await {
8991                                error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
8992                                fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8993                                    &runtime.spawn_events, format!("registration_timeout: {error}"),
8994                                    &runtime.failure_context(true));
8995                                child = None;
8996                            }
8997                            // The ordinary exit handler records the child's exit,
8998                            // spends crash budget, and schedules recovery.
8999                            continue;
9000                        }
9001                    }
9002                    if health_probe.due() {
9003                        run_health_probe_cycle(
9004                            &spec,
9005                            &runtime,
9006                            &registry,
9007                            &process_liveness,
9008                            &snapshot,
9009                            &mut child,
9010                        ).await;
9011                        if child.is_none() {
9012                            runtime.start_requester = None;
9013                            runtime.pending_activity_restart = false;
9014                        }
9015                        if child.is_some() {
9016                            health_probe.schedule_next(&spec, runtime.health.cadence);
9017                        }
9018                    }
9019                }
9020            }
9021        } else if let Some(pending) = pending_respawn.clone() {
9022            tokio::select! {
9023                _ = runtime.child_roster.closed() => {
9024                    pending_respawn = None;
9025                    let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9026                        state.state = ModuleState::Stopped;
9027                        state.respawn_pending = false;
9028                    });
9029                    containment::terminate(&runtime.child_roster.containment).await;
9030                }
9031                _ = async {
9032                    sleep_until(pending.deadline).await;
9033                    if let Some(delivery) = &runtime.config_delivery {
9034                        delivery.unfenced(&spec.module_id).await;
9035                    }
9036                } => {
9037                    pending_respawn = None;
9038                    let failure = pending.failure_context(&runtime);
9039                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
9040                    // A command handled below while the backoff elapsed may
9041                    // have stopped the module; never respawn past an operator's
9042                    // disable or drain.
9043                    if !respawn_still_pending(&snapshot) {
9044                        continue;
9045                    }
9046                    // The daemon began shutting down during the backoff: the
9047                    // spawn would be refused anyway, and refusing it here
9048                    // leaves the module stopped instead of reporting a
9049                    // failed restart.
9050                    if runtime.child_roster.is_closed() {
9051                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9052                            state.state = ModuleState::Stopped;
9053                        });
9054                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
9055                        continue;
9056                    }
9057                    if let Err(err) = release_dead_registration(
9058                        &registry,
9059                        runtime.forwarding.as_deref(),
9060                        &snapshot,
9061                        &spec.module_id,
9062                        &failure,
9063                    ).await {
9064                        record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
9065                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
9066                        continue;
9067                    }
9068
9069                    if matches!(pending.kind, RespawnKind::Reload) {
9070                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
9071                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
9072                        if matches!(result, Err(SuperviseError::ConfigDeliveryDeferred { .. })) {
9073                            *runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()) = reply;
9074                            continue;
9075                        }
9076                        if result.is_ok() && runtime.pending_activity_restart {
9077                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
9078                        }
9079                        runtime.pending_activity_restart = false;
9080                        if let Some(reply) = reply { let _ = reply.send(result); }
9081                        continue;
9082                    }
9083                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
9084                    match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
9085                        Ok(next_child) => {
9086                            child = Some(next_child);
9087                            if runtime.pending_activity_restart {
9088                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
9089                                runtime.pending_activity_restart = false;
9090                            }
9091                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
9092                        }
9093                        Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9094                            pending_respawn = Some(PendingRespawn { deadline:Instant::now()+CONFIG_ADMISSION_RETRY_DELAY, ..pending });
9095                            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
9096                        }
9097                        Err(SuperviseError::ConfigHeld { .. }) => {
9098                            let _ = hold_config_spawn(&spec, &runtime, &snapshot, &failure);
9099                            runtime.pending_activity_restart = false;
9100                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9101                        }
9102                        Err(err) => {
9103                            fail_snapshot(&snapshot, Some(&spec.module_id), None, &failure);
9104                            record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
9105                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9106                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
9107                        }
9108                    }
9109                }
9110                command = commands.recv() => {
9111                    let Some(command) = command else {
9112                        containment::terminate(&runtime.child_roster.containment).await;
9113                        return;
9114                    };
9115                    if !handle_supervisor_command(
9116                        command,
9117                        &mut spec,
9118                        &mut runtime,
9119                        &registry,
9120                        &process_liveness,
9121                        &snapshot,
9122                        &mut child,
9123                        &mut commands,
9124                        &mut requeued,
9125                    ).await {
9126                        containment::terminate(&runtime.child_roster.containment).await;
9127                        return;
9128                    }
9129                    // Reconcile the pending respawn with what the command did:
9130                    // a start may already have spawned a fresh child,
9131                    // while a disable or drain moved the snapshot out of the
9132                    // state the respawn was counting down from.
9133                    if child.is_some() || !respawn_still_pending(&snapshot) {
9134                        pending_respawn = None;
9135                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
9136                    }
9137                }
9138            }
9139        } else {
9140            let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
9141                (state.enabled && state.state == ModuleState::Failed)
9142                    .then_some(state.failed_retry.pending)
9143                    .flatten()
9144            });
9145            let command = if lock_snapshot(&snapshot)
9146                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
9147            {
9148                tokio::select! {
9149                    command = commands.recv() => command,
9150                    _ = previous_instance_recheck.tick() => {
9151                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
9152                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
9153                            if enabled && !runtime.child_roster.is_closed() {
9154                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
9155                                match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
9156                                    Ok(next_child) => child = Some(next_child),
9157                                    Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9158                                        let _ = defer_config_spawn(&spec, &runtime, &snapshot, RespawnKind::Spawn, &runtime.failure_context(true));
9159                                    }
9160                                    Err(SuperviseError::ConfigHeld { .. }) => {
9161                                        let _ = hold_config_spawn(&spec, &runtime, &snapshot, &runtime.failure_context(true));
9162                                    }
9163                                    Err(error) => {
9164                                        fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
9165                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
9166                                    }
9167                                }
9168                            } else {
9169                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9170                                    state.waiting_for_previous_pid = None;
9171                                    state.waiting_for_previous_detail = None;
9172                                });
9173                            }
9174                        }
9175                        continue;
9176                    }
9177                }
9178            } else {
9179                tokio::select! {
9180                    biased;
9181                    command = commands.recv() => command,
9182                    _ = async {
9183                        match slow_retry {
9184                            Some(retry) => sleep_until(retry.deadline).await,
9185                            None => std::future::pending().await,
9186                        }
9187                    } => {
9188                        let retry = slow_retry.expect("only a scheduled slow retry can wake");
9189                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9190                            state.failed_retry.pending = None;
9191                        });
9192                        if runtime.child_roster.is_closed() {
9193                            continue;
9194                        }
9195                        info!(module_id = %spec.module_id, attempt = retry.attempt,
9196                            delay_secs = retry.delay.as_secs(), "starting failed module retry");
9197                        if let Err(error) = set_child_enabled(
9198                            &spec, &runtime, &registry, &process_liveness, &snapshot, &mut child, true,
9199                            &runtime.failure_context(true),
9200                        ).await {
9201                            error!(module_id = %spec.module_id, %error, "failed module retry could not start");
9202                        }
9203                        continue;
9204                    }
9205                }
9206            };
9207            let Some(command) = command else {
9208                containment::terminate(&runtime.child_roster.containment).await;
9209                return;
9210            };
9211            if !handle_supervisor_command(
9212                command,
9213                &mut spec,
9214                &mut runtime,
9215                &registry,
9216                &process_liveness,
9217                &snapshot,
9218                &mut child,
9219                &mut commands,
9220                &mut requeued,
9221            )
9222            .await
9223            {
9224                containment::terminate(&runtime.child_roster.containment).await;
9225                return;
9226            }
9227        }
9228    }
9229}
9230
9231// Preserve the deadline until the future is polled. Starting a relative sleep
9232// after the executor yields would restart part of the countdown.
9233async fn supervisor_sleep(wake_at: Option<Instant>) {
9234    match wake_at {
9235        Some(deadline) => sleep_until(deadline).await,
9236        None => std::future::pending().await,
9237    }
9238}
9239
9240fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
9241    info!(
9242        module_id,
9243        restart_in_window = schedule.restart_in_window,
9244        delay_ms = schedule.delay.as_millis() as u64,
9245        "respawning after crash"
9246    );
9247}
9248
9249/// Whether the respawn a backoff was counting down to is still wanted. A
9250/// disable or drain handled while the backoff elapsed moves the snapshot out
9251/// of `Restarting`, and the operator's stop must win over the pending respawn,
9252/// so every sleep-then-spawn path re-validates against the live snapshot
9253/// instead of assuming the state it left behind still holds.
9254fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
9255    matches!(
9256        lock_snapshot(snapshot),
9257        Ok(state) if state.enabled && state.state == ModuleState::Restarting
9258    )
9259}
9260
9261enum NextAction {
9262    Stop {
9263        registration_released: bool,
9264    },
9265    Restart {
9266        schedule: Option<CrashRestartSchedule>,
9267    },
9268}
9269
9270#[allow(clippy::too_many_arguments)]
9271async fn handle_supervisor_command(
9272    command: SupervisorCommand,
9273    spec: &mut ModuleSpec,
9274    runtime: &mut SupervisorRuntimeConfig,
9275    registry: &Arc<Registry>,
9276    process_liveness: &SupervisorProcessLiveness,
9277    snapshot: &SharedSnapshot,
9278    child: &mut Option<SupervisedChild>,
9279    commands: &mut mpsc::Receiver<SupervisorCommand>,
9280    requeued: &mut VecDeque<SupervisorCommand>,
9281) -> bool {
9282    if let SupervisorCommand::SetEnabled {
9283        enabled: true,
9284        reset_budget: false,
9285        reply,
9286        ..
9287    } = command
9288    {
9289        let parked = lock_snapshot(snapshot).ok().and_then(|state| {
9290            (state.config_hold_parked
9291                && state.enabled
9292                && !state.process_alive
9293                && child.is_none()
9294                && state.state == ModuleState::Stopped)
9295                .then(|| {
9296                    (
9297                        state.config_hold_daemon_initiated,
9298                        state.config_hold_approval.clone(),
9299                    )
9300                })
9301        });
9302        // Clearing a configuration-delivery block must not restart a child that
9303        // exited on its own or exhausted its crash budget. Resume only an actor
9304        // stopped by a configuration hold; otherwise leave its slow retry alone.
9305        let result = if let Some((daemon_initiated, approval)) = parked {
9306            runtime.recovery_is_daemon = daemon_initiated;
9307            let failure = FailureContext {
9308                approval,
9309                ..runtime.failure_context(daemon_initiated)
9310            };
9311            let result = set_child_enabled_with_budget_policy(
9312                spec,
9313                runtime,
9314                registry,
9315                process_liveness,
9316                snapshot,
9317                child,
9318                true,
9319                &failure,
9320                false,
9321            )
9322            .await;
9323            if let Err(error) = &result {
9324                record_approved_attempt_failure(spec, snapshot, &failure, error);
9325            }
9326            result
9327        } else {
9328            Ok(false)
9329        };
9330        let _ = reply.send(result);
9331        return true;
9332    }
9333    let blocked = runtime
9334        .config_delivery
9335        .as_ref()
9336        .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
9337        .filter(|error| error.code != "config_delivery_deferred")
9338        .map(|error| {
9339            config_delivery::gate_hold_error(spec, error, runtime.config_delivery.as_ref())
9340        })
9341        .or_else(|| recovery_block_error(spec, &runtime.child_roster))
9342        .or_else(|| {
9343            runtime
9344                .child_roster
9345                .previous_instance(&spec.module_id)
9346                .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
9347        });
9348    let command = if let Some(error) = blocked {
9349        match command {
9350            SupervisorCommand::Restart { reply, .. } | SupervisorCommand::Reload { reply, .. } => {
9351                let _ = reply.send(Err(error));
9352                return true;
9353            }
9354            SupervisorCommand::Swap { reply, .. } => {
9355                let _ = reply.send(Err(error));
9356                return true;
9357            }
9358            SupervisorCommand::ConfigApplyRestart { .. } => return true,
9359            SupervisorCommand::SetEnabled {
9360                enabled: true,
9361                requester,
9362                reply,
9363                ..
9364            } => {
9365                if matches!(error, SuperviseError::ConfigHeld { .. })
9366                    && lock_snapshot(snapshot).is_ok_and(|state| !state.enabled)
9367                    && runtime
9368                        .config_delivery
9369                        .as_ref()
9370                        .is_some_and(|delivery| delivery.0.can_stage_unstamp_enable())
9371                {
9372                    // Stage only the pending unstamp's desired state. A stamped
9373                    // unapproved actor stays refused; admission release starts
9374                    // an unstamped actor only after reconciliation completes.
9375                    let mut changed = false;
9376                    let result = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9377                        changed = !state.enabled;
9378                        state.enabled = true;
9379                        if child.is_none() && state.waiting_for_previous_pid.is_none() {
9380                            state.state = ModuleState::Starting;
9381                            state.failed_retry.cancel();
9382                        }
9383                    })
9384                    .and_then(|_| {
9385                        if child.is_none()
9386                            && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none()
9387                        {
9388                            hold_config_spawn(
9389                                spec,
9390                                runtime,
9391                                snapshot,
9392                                &runtime.failure_context(false),
9393                            )?;
9394                        }
9395                        Ok(changed)
9396                    });
9397                    if result.is_ok() && changed {
9398                        runtime.start_requester = requester;
9399                    }
9400                    let _ = reply.send(result);
9401                    return true;
9402                }
9403                let _ = reply.send(Err(error));
9404                return true;
9405            }
9406            other => other,
9407        }
9408    } else {
9409        command
9410    };
9411    // A refused start or restart leaves the Failed actor's existing recovery
9412    // deadline intact. Cancel only after the command passes admission.
9413    if matches!(
9414        command,
9415        SupervisorCommand::Drain { .. }
9416            | SupervisorCommand::Retire { .. }
9417            | SupervisorCommand::SetEnabled { .. }
9418            | SupervisorCommand::Restart { .. }
9419    ) {
9420        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9421            state.failed_retry.cancel();
9422        });
9423    }
9424    match command {
9425        SupervisorCommand::ApprovedConfiguration { approved, reply } => {
9426            let result = async {
9427                let next = approved.module_entry(&spec.module_id).map_err(|error| {
9428                    SuperviseError::InvalidSpec { reason: error.to_string() }
9429                })?;
9430                let retry_approved_failure = {
9431                    let mut state = lock_snapshot(snapshot)?;
9432                    if let Some((generation, detail)) = &state.approved_start_failure {
9433                        if *generation == approved.identity() && next.enabled {
9434                            return Err(SuperviseError::Spawn {
9435                                program: spec.program.clone(),
9436                                source: std::io::Error::other(detail.clone()),
9437                                cgroup_path: None,
9438                            });
9439                        }
9440                    }
9441                    let retry = state.approved_start_failure.is_some();
9442                    state.approved_start_failure = None;
9443                    retry
9444                };
9445                stage_supervisor_configuration(
9446                    spec, runtime, snapshot, next.module_spec(), next.health,
9447                    next.drain_timeout_ms,
9448                );
9449                runtime.restart_policy = next.restart;
9450                runtime.register_timeout = next.register_timeout;
9451                if let Some(active) = child.as_mut()
9452                    .filter(|child| child.registration_deadline.is_some())
9453                {
9454                    active.registration_deadline = Some(active.spawned_at + next.register_timeout);
9455                }
9456                let enabled = lock_snapshot(snapshot)?.enabled;
9457                // An approval retries a changed failed entry or its own earlier
9458                // failed start, not an unrelated crash-budget failure. For an
9459                // unchanged entry, reconciled means the approved configuration
9460                // is effective; it does not claim the process has recovered.
9461                // A terminal gate stop is different: a new approval owns its
9462                // recovery even when the daemon entry itself did not change.
9463                let retry_gate_stopped = lock_snapshot(snapshot)?
9464                    .needs_approved_gate_restart(&approved.identity());
9465                let retry_failed = enabled && child.is_none()
9466                    && ((lock_snapshot(snapshot)?.state == ModuleState::Failed
9467                        && (retry_approved_failure || approved.changes_entry(&spec.module_id)))
9468                        || retry_gate_stopped);
9469                if retry_failed {
9470                    // This approval owns a new start, so the failed attempt's
9471                    // daemon retry must not survive either result of that start.
9472                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
9473                        state.failed_retry.cancel();
9474                        if retry_gate_stopped {
9475                            state.stop_reason = None;
9476                            state.config_exit_failed_approval = None;
9477                        }
9478                    })?;
9479                }
9480                if enabled != next.enabled || retry_failed {
9481                    let failure = FailureContext {
9482                        approval: next.enabled.then(|| approved.identity()),
9483                        ..runtime.failure_context(false)
9484                    };
9485                    let held = next.enabled && runtime.config_delivery.as_ref()
9486                        .is_some_and(|delivery| delivery.0.has_recorded_admission_hold(&spec.module_id)
9487                            || delivery.0.check_module_hold(&spec.module_id).is_err_and(|error| error.code == "config_delivery_deferred"));
9488                    if held {
9489                        // Preserve module-delivery failure/recovery holds. The
9490                        // admission release starts the actor when its own hold ends.
9491                        reset_restart_count(snapshot, &spec.module_id)?;
9492                        update_snapshot(snapshot, Some(&spec.module_id), |state| {
9493                            state.enabled = true;
9494                            if child.is_none() && state.waiting_for_previous_pid.is_none() {
9495                                state.state = ModuleState::Starting;
9496                            }
9497                        })?;
9498                        if child.is_none() && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none() {
9499                            hold_config_spawn(spec, runtime, snapshot, &failure)?;
9500                        }
9501                    } else {
9502                        match set_child_enabled_with_budget_policy(
9503                            spec, runtime, registry, process_liveness, snapshot, child,
9504                            next.enabled, &failure, true,
9505                        ).await {
9506                            Ok(_) => {},
9507                            Err(SuperviseError::ConfigHeld { .. }) if next.enabled => {},
9508                            Err(error) => return Err(error),
9509                        }
9510                        if !next.enabled {
9511                            containment::terminate(&runtime.child_roster.containment).await;
9512                        }
9513                    }
9514                }
9515                if (child.as_ref().is_some_and(|child| child.registration_spawn.is_some() && child.admission_base.is_none())
9516                        || (child.is_none() && lock_snapshot(snapshot)?.state == ModuleState::Restarting
9517                            && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.stamp_channel_cutover_pending())))
9518                    && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.module_is_gated(&spec.module_id))
9519                {
9520                    let failure = FailureContext {
9521                        approval: Some(approved.identity()),
9522                        ..runtime.failure_context(false)
9523                    };
9524                    // A stamp cannot adopt an unverified legacy process, even
9525                    // when its approved replacement no longer speaks subc.
9526                    // Stopping that process spends no crash restart budget.
9527                    if child.is_some() {
9528                        restart_child_with_budget_policy(spec, runtime, registry,
9529                            process_liveness, snapshot, child, runtime.drain_timeout, false).await?;
9530                    }
9531                    // The stamp is already committed. A deferred or held launch
9532                    // belongs to the normal retry/hold loop, not a failed apply.
9533                    update_snapshot(snapshot, Some(&spec.module_id), |state| state.respawn_pending = false)?;
9534                    match spawn_and_mark_running_async(spec, runtime, snapshot).await {
9535                        Ok(replacement) => *child = Some(replacement),
9536                        Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9537                            defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, &failure)?;
9538                        }
9539                        Err(SuperviseError::ConfigHeld { .. }) => hold_config_spawn(spec, runtime, snapshot, &failure)?,
9540                        Err(error) => {
9541                            fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
9542                            return Err(error);
9543                        }
9544                    }
9545                }
9546                info!(module_id=%spec.module_id, config_generation=approved.generation(), "approved daemon entry reconciled");
9547                Ok(())
9548            }
9549            .await;
9550            if let Err(error) = &result {
9551                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9552                    if state.enabled
9553                        && state.state == ModuleState::Failed
9554                        && state
9555                            .approved_start_failure
9556                            .as_ref()
9557                            .is_none_or(|(generation, _)| *generation != approved.identity())
9558                    {
9559                        state.approved_start_failure =
9560                            Some((approved.identity(), error.to_string()));
9561                    }
9562                });
9563            }
9564            let _ = reply.send(result);
9565            true
9566        }
9567        SupervisorCommand::ConfigPolicyProbe {
9568            connection,
9569            generation,
9570            attempt,
9571        } => {
9572            let Some(delivery) = &runtime.config_delivery else {
9573                return true;
9574            };
9575            let Ok(gate) = delivery.0.module_gate(&spec.module_id) else {
9576                return true;
9577            };
9578            let guard = gate.lock().await;
9579            let pending = delivery
9580                .0
9581                .on_blocking(move |owner| {
9582                    owner.policy_probe_pending(connection, generation, attempt)
9583                })
9584                .await
9585                .unwrap_or(false)
9586                && child.is_some()
9587                && registry
9588                    .get_module(&spec.module_id)
9589                    .ok()
9590                    .flatten()
9591                    .is_some_and(|registration| registration.connection_id == connection);
9592            drop(guard);
9593            if pending {
9594                run_health_probe_cycle(spec, runtime, registry, process_liveness, snapshot, child)
9595                    .await;
9596            }
9597            true
9598        }
9599        SupervisorCommand::ConfigApplyRestart {
9600            generation,
9601            supersedes_swap,
9602        } => {
9603            let Some(owner) = runtime
9604                .config_delivery
9605                .as_ref()
9606                .map(|delivery| delivery.0.clone())
9607            else {
9608                return true;
9609            };
9610            let Ok(gate) = owner.module_gate(&spec.module_id) else {
9611                return true;
9612            };
9613            let _guard = gate.lock().await;
9614            let spawn = child
9615                .as_ref()
9616                .and_then(|child| child.config_delivery.clone());
9617            let module = spec.module_id.clone();
9618            let current = owner
9619                .on_blocking(move |owner| {
9620                    Ok((supersedes_swap
9621                        || owner
9622                            .pending_config_restarts()?
9623                            .contains(&(module.clone(), generation)))
9624                        && owner.requires_fresher_spawn(&module, spawn.as_ref())?
9625                            == Some(generation))
9626                })
9627                .await
9628                .unwrap_or(false);
9629            if !current
9630                || !lock_snapshot(snapshot).is_ok_and(|state| {
9631                    state.enabled && !state.respawn_pending && state.process_alive
9632                })
9633            {
9634                return true;
9635            }
9636            drop(_guard);
9637            info!(module_id=%spec.module_id, config_generation=generation, "approved configuration restart preserves the crash budget");
9638            if let Err(error) = restart_child_with_budget_policy(
9639                spec,
9640                runtime,
9641                registry,
9642                process_liveness,
9643                snapshot,
9644                child,
9645                runtime.drain_timeout,
9646                false,
9647            )
9648            .await
9649            {
9650                warn!(module_id=%spec.module_id, %error, "approved configuration restart could not be scheduled");
9651            }
9652            true
9653        }
9654        SupervisorCommand::ConfigExitHold { reply } => {
9655            // A notification owns no start, backoff, reload reply or previous-
9656            // instance wait. Only an idle, registered initial start has no work
9657            // to preserve; its initialized startup context can be parked here.
9658            let idle_start = child.is_none()
9659                && lock_snapshot(snapshot).is_ok_and(|state| {
9660                    state.enabled
9661                        && state.state == ModuleState::Starting
9662                        && !state.process_alive
9663                        && !state.respawn_pending
9664                        && state.waiting_for_previous_pid.is_none()
9665                        && state.waiting_for_previous_detail.is_none()
9666                        && state.failed_retry.pending.is_none()
9667                })
9668                && runtime
9669                    .scheduled_respawn
9670                    .lock()
9671                    .unwrap_or_else(|p| p.into_inner())
9672                    .is_none()
9673                && runtime
9674                    .deferred_reload_reply
9675                    .lock()
9676                    .unwrap_or_else(|p| p.into_inner())
9677                    .is_none();
9678            let result = if idle_start {
9679                try_park_config_attempt(
9680                    spec,
9681                    runtime,
9682                    snapshot,
9683                    &runtime.failure_context(runtime.recovery_is_daemon),
9684                    None,
9685                )
9686                .map(|_| ())
9687            } else {
9688                Ok(())
9689            };
9690            let _ = reply.send(result);
9691            true
9692        }
9693        SupervisorCommand::ConfigDeliveryFailed { connection, hold } => {
9694            if registry
9695                .get_module(&spec.module_id)
9696                .ok()
9697                .flatten()
9698                .is_none_or(|registration| registration.connection_id != connection)
9699            {
9700                return true;
9701            }
9702            runtime.recovery_is_daemon = true;
9703            // With a recoverable generation, delivery-failure recovery spends
9704            // one crash-budget entry: the module rejected its configuration.
9705            // Approved configuration restarts spend none.
9706            if hold {
9707                let notice = begin_forwarding_drain_if_configured(
9708                    spec,
9709                    runtime,
9710                    registry,
9711                    snapshot,
9712                    Some(true),
9713                    RouteCloseReason::Disable,
9714                )
9715                .await;
9716                if let Ok(notice) = notice {
9717                    let drained = drain_optional_child(
9718                        &spec.module_id,
9719                        spec.protocol,
9720                        notice,
9721                        registry,
9722                        runtime.forwarding.as_deref(),
9723                        snapshot,
9724                        &runtime.terminal_ring,
9725                        &runtime.spawn_events,
9726                        child,
9727                        runtime.drain_timeout,
9728                        ModuleState::Failed,
9729                        Some(true),
9730                        &runtime.failure_context(true),
9731                    )
9732                    .await;
9733                    if drained.is_ok() {
9734                        match try_park_config_attempt(
9735                            spec,
9736                            runtime,
9737                            snapshot,
9738                            &runtime.failure_context(true),
9739                            None,
9740                        ) {
9741                            Ok(true) => {}
9742                            Ok(false) => {
9743                                if let Err(error) = budgeted_restart_child(
9744                                    spec,
9745                                    runtime,
9746                                    registry,
9747                                    process_liveness,
9748                                    snapshot,
9749                                    child,
9750                                    SupervisorHealthStatus::Unknown,
9751                                    Some("configuration delivery failed after its hold ended"),
9752                                    unix_ms_now(),
9753                                    false,
9754                                )
9755                                .await
9756                                {
9757                                    warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9758                                }
9759                            }
9760                            Err(error) => {
9761                                warn!(module_id=%spec.module_id, %error, "configuration failure could not be parked")
9762                            }
9763                        }
9764                    }
9765                }
9766            } else if let Err(error) = budgeted_restart_child(
9767                spec,
9768                runtime,
9769                registry,
9770                process_liveness,
9771                snapshot,
9772                child,
9773                SupervisorHealthStatus::Unknown,
9774                Some("configuration delivery failed"),
9775                unix_ms_now(),
9776                false,
9777            )
9778            .await
9779            {
9780                warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9781            }
9782            true
9783        }
9784        SupervisorCommand::Drain { requester, reply } => {
9785            // A plain stop runs no forwarding drain, so nothing reaches the
9786            // module over its connection before the wait: ask by signal.
9787            let result = drain_optional_child(
9788                &spec.module_id,
9789                spec.protocol,
9790                StopNotice::NotSent,
9791                registry,
9792                runtime.forwarding.as_deref(),
9793                snapshot,
9794                &runtime.terminal_ring,
9795                &runtime.spawn_events,
9796                child,
9797                runtime.drain_timeout,
9798                ModuleState::Stopped,
9799                None,
9800                &FailureContext::operator(),
9801            )
9802            .await;
9803            containment::terminate(&runtime.child_roster.containment).await;
9804            let registration_released = result.is_ok();
9805            if registration_released {
9806                runtime.record_activity_transition(
9807                    "subc.module_stopped",
9808                    &spec.module_id,
9809                    requester,
9810                );
9811            }
9812            let _ = reply.send(result);
9813            if registration_released {
9814                process_liveness.untrack_if_current(&spec.module_id, snapshot);
9815            }
9816            false
9817        }
9818        SupervisorCommand::Retire { requester, reply } => {
9819            let result = async {
9820                let stop_notice = begin_forwarding_drain_if_configured(
9821                    spec,
9822                    runtime,
9823                    registry,
9824                    snapshot,
9825                    None,
9826                    RouteCloseReason::Disable,
9827                )
9828                .await?;
9829                drain_optional_child(
9830                    &spec.module_id,
9831                    spec.protocol,
9832                    stop_notice,
9833                    registry,
9834                    runtime.forwarding.as_deref(),
9835                    snapshot,
9836                    &runtime.terminal_ring,
9837                    &runtime.spawn_events,
9838                    child,
9839                    runtime.drain_timeout,
9840                    ModuleState::Stopped,
9841                    None,
9842                    &FailureContext::operator(),
9843                )
9844                .await
9845            }
9846            .await;
9847            containment::terminate(&runtime.child_roster.containment).await;
9848            let registration_released = result.is_ok();
9849            if registration_released {
9850                runtime.record_activity_transition(
9851                    "subc.module_stopped",
9852                    &spec.module_id,
9853                    requester,
9854                );
9855            }
9856            let _ = reply.send(result);
9857            if registration_released {
9858                process_liveness.untrack_if_current(&spec.module_id, snapshot);
9859            }
9860            false
9861        }
9862        SupervisorCommand::Restart {
9863            requester,
9864            drain_timeout_ms,
9865            received_at_generation,
9866            queued_at,
9867            reply,
9868        } => {
9869            // Without this line a restart that waited in the queue (behind a
9870            // health probe cycle or another command) was invisible: the log
9871            // showed only the drain timing out, minutes after the operator's call.
9872            info!(
9873                module_id = %spec.module_id,
9874                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
9875                "restart command dequeued"
9876            );
9877            // ACK AT INITIATION, not completion. The blocking form deadlocked any
9878            // caller whose own request lane rides the module being restarted: the
9879            // caller's in-flight request keeps the drain from quiescing, the drain
9880            // keeps the restart from completing, and the completion keeps the reply
9881            // from releasing the caller — so the drain always timed out and cut the
9882            // initiator with a GOODBYE, even on a healthy module. Replying once the
9883            // restart is validated lets a self-lane caller settle, which is exactly
9884            // what makes the drain succeed. Completion is observable via
9885            // supervisor.list / module status; a post-ack failure lands the module
9886            // in a visible terminal state below rather than in a reply nobody can
9887            // receive.
9888            let validation = match lock_snapshot(snapshot) {
9889                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
9890                    module_id: spec.module_id.clone(),
9891                }),
9892                Ok(_) => Ok(()),
9893                Err(err) => Err(err),
9894            };
9895            let initiated = validation.is_ok();
9896            let _ = reply.send(validation);
9897            // A restart asks for a fresh process. Commands run one at a time,
9898            // so a restart queued behind another restart (two operator calls
9899            // in quick succession) is dequeued the moment the first one has
9900            // spawned its replacement -- before that process has sent HELLO.
9901            // Running it would drain and kill the process the first restart
9902            // just produced, which is the opposite of what both callers asked
9903            // for. If a process spawned after this request was received is
9904            // still supervised, the request is already satisfied. Not when the
9905            // configuration changed since that spawn: then the newer process
9906            // predates the spec this restart may exist to apply.
9907            // Generation publication can lag a child that is already serving
9908            // while its identity is recorded. Only process creation after request
9909            // receipt satisfies a restart, not publication after receipt.
9910            let satisfied_by_generation = if initiated
9911                && child
9912                    .as_ref()
9913                    .is_some_and(|child| child.spawned_at > queued_at)
9914            {
9915                lock_snapshot(snapshot).ok().and_then(|state| {
9916                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
9917                })
9918            } else {
9919                None
9920            };
9921            let satisfied_by_pending = initiated
9922                && child.is_none()
9923                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
9924                    // No child exists yet, so this replacement will necessarily
9925                    // be created after the queued request, unlike a published
9926                    // generation whose process may already have been serving.
9927                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
9928                    if pending {
9929                        state.coalesced_restart_pending = true;
9930                    }
9931                    pending
9932                });
9933            if satisfied_by_pending {
9934                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
9935            } else if let Some(generation) = satisfied_by_generation {
9936                info!(
9937                    module_id = %spec.module_id,
9938                    received_at_generation,
9939                    "restart already satisfied by generation {generation}; not restarting again"
9940                );
9941            } else if initiated {
9942                // Precedence: this restart's operator override, else the module's
9943                // configured budget (already resolved into the runtime).
9944                let drain_timeout = drain_timeout_ms
9945                    .map(Duration::from_millis)
9946                    .unwrap_or(runtime.drain_timeout);
9947                if let Err(err) = restart_child(
9948                    spec,
9949                    runtime,
9950                    registry,
9951                    process_liveness,
9952                    snapshot,
9953                    child,
9954                    drain_timeout,
9955                )
9956                .await
9957                {
9958                    warn!(
9959                        module_id = %spec.module_id,
9960                        error = %err,
9961                        "operator restart failed after initiation ack; module state carries the outcome"
9962                    );
9963                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9964                        state.state = ModuleState::Failed;
9965                        clear_current_process_facts(state);
9966                    });
9967                } else {
9968                    runtime.start_requester = requester;
9969                    runtime.pending_activity_restart = true;
9970                }
9971            }
9972            true
9973        }
9974        SupervisorCommand::Reload { requester, reply } => {
9975            let result =
9976                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
9977            if result.is_ok() {
9978                runtime.start_requester = requester;
9979                runtime.pending_activity_restart = true;
9980            }
9981            if result.is_ok()
9982                && runtime
9983                    .scheduled_respawn
9984                    .lock()
9985                    .unwrap_or_else(|p| p.into_inner())
9986                    .as_ref()
9987                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
9988            {
9989                *runtime
9990                    .deferred_reload_reply
9991                    .lock()
9992                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
9993            } else {
9994                let _ = reply.send(result);
9995            }
9996            true
9997        }
9998        SupervisorCommand::SetEnabled {
9999            enabled,
10000            requester,
10001            reply,
10002            reset_budget,
10003        } => {
10004            let previous_requester = runtime.start_requester.clone();
10005            if enabled {
10006                runtime.start_requester = requester.clone();
10007            }
10008            let result = set_child_enabled_with_budget_policy(
10009                spec,
10010                runtime,
10011                registry,
10012                process_liveness,
10013                snapshot,
10014                child,
10015                enabled,
10016                &runtime.failure_context(false),
10017                reset_budget,
10018            )
10019            .await;
10020            if !enabled {
10021                containment::terminate(&runtime.child_roster.containment).await;
10022            }
10023            if matches!(result, Ok(true)) {
10024                runtime.pending_activity_restart = false;
10025                if enabled {
10026                    runtime.start_requester = requester;
10027                } else {
10028                    runtime.start_requester = None;
10029                    runtime.record_activity_transition(
10030                        "subc.module_disabled",
10031                        &spec.module_id,
10032                        requester,
10033                    );
10034                }
10035            } else {
10036                runtime.start_requester = previous_requester;
10037            }
10038            let _ = reply.send(result);
10039            true
10040        }
10041        SupervisorCommand::UpdateConfiguration {
10042            spec: next_spec,
10043            health,
10044            drain_timeout_ms,
10045            reply,
10046        } => {
10047            if runtime
10048                .config_delivery
10049                .as_ref()
10050                .is_some_and(|delivery| delivery.0.file_configuration_is_gated(&spec.module_id))
10051            {
10052                let _ = reply.send(Err(SuperviseError::ConfigHeld {
10053                    module_id: spec.module_id.clone(),
10054                    code: "config_gated".into(),
10055                    detail: "live file reconfiguration cannot replace stamped authority".into(),
10056                }));
10057                return true;
10058            }
10059            stage_supervisor_configuration(
10060                spec,
10061                runtime,
10062                snapshot,
10063                next_spec,
10064                health,
10065                drain_timeout_ms,
10066            );
10067            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10068                state.configuration_updated_since_spawn = true;
10069            });
10070            let _ = reply.send(Ok(()));
10071            true
10072        }
10073        SupervisorCommand::Swap {
10074            requester,
10075            ready_timeout,
10076            reply,
10077        } => {
10078            if spec.carry_containment
10079                || containment::current(&runtime.child_roster.containment).is_some()
10080            {
10081                let _ = reply.send(Err(SuperviseError::InvalidSpec {
10082                    reason: "a carried container cannot overlap a swap candidate".to_owned(),
10083                }));
10084                return true;
10085            }
10086            let mut swap_runtime = runtime.clone();
10087            swap_runtime.start_requester = requester;
10088            let before = lock_snapshot(snapshot)
10089                .ok()
10090                .map(|state| state.spawn_generation);
10091            let end = swap::run_swap(
10092                spec,
10093                &swap_runtime,
10094                registry,
10095                process_liveness,
10096                snapshot,
10097                child,
10098                commands,
10099                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
10100                reply,
10101            )
10102            .await;
10103            if before
10104                != lock_snapshot(snapshot)
10105                    .ok()
10106                    .map(|state| state.spawn_generation)
10107            {
10108                runtime.start_requester = swap_runtime.start_requester;
10109            }
10110            requeued.extend(end.requeue);
10111            true
10112        }
10113    }
10114}
10115
10116fn stage_supervisor_configuration(
10117    spec: &mut ModuleSpec,
10118    runtime: &mut SupervisorRuntimeConfig,
10119    snapshot: &SharedSnapshot,
10120    next_spec: ModuleSpec,
10121    health: HealthConfig,
10122    drain_timeout_ms: Option<u64>,
10123) {
10124    if let Some(handle) = &runtime.supervisor_handle {
10125        handle.apply_identity_configuration(&next_spec);
10126    }
10127    let launch_changed = *spec != next_spec;
10128    *spec = next_spec;
10129    if launch_changed {
10130        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10131            state.configuration_updated_since_spawn = true
10132        });
10133    }
10134    let health_changed = runtime.health != health;
10135    runtime.health = health;
10136    // Reset the cadence and old endpoint's failure streak on a live
10137    // health-policy change rather than waiting for its old deadline.
10138    if health_changed {
10139        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10140            state.health = ModuleHealthStatus::default();
10141            state.last_probe_evidence = None;
10142        });
10143    }
10144    runtime.drain_timeout = drain_timeout_ms
10145        .map(Duration::from_millis)
10146        .unwrap_or(runtime.default_drain_timeout);
10147    *runtime
10148        .effective_drain_timeout
10149        .lock()
10150        .unwrap_or_else(|p| p.into_inner()) = runtime.drain_timeout;
10151}
10152
10153async fn restart_child(
10154    spec: &ModuleSpec,
10155    runtime: &SupervisorRuntimeConfig,
10156    registry: &Registry,
10157    process_liveness: &SupervisorProcessLiveness,
10158    snapshot: &SharedSnapshot,
10159    child: &mut Option<SupervisedChild>,
10160    drain_timeout: Duration,
10161) -> Result<(), SuperviseError> {
10162    restart_child_with_budget_policy(
10163        spec,
10164        runtime,
10165        registry,
10166        process_liveness,
10167        snapshot,
10168        child,
10169        drain_timeout,
10170        true,
10171    )
10172    .await
10173}
10174
10175#[allow(clippy::too_many_arguments)]
10176async fn restart_child_with_budget_policy(
10177    spec: &ModuleSpec,
10178    runtime: &SupervisorRuntimeConfig,
10179    registry: &Registry,
10180    process_liveness: &SupervisorProcessLiveness,
10181    snapshot: &SharedSnapshot,
10182    child: &mut Option<SupervisedChild>,
10183    drain_timeout: Duration,
10184    reset_budget: bool,
10185) -> Result<(), SuperviseError> {
10186    // Restart cycles a running module; it must not silently start a disabled one.
10187    if !lock_snapshot(snapshot)?.enabled {
10188        return Err(SuperviseError::Disabled {
10189            module_id: spec.module_id.clone(),
10190        });
10191    }
10192    let stop_notice = begin_forwarding_drain_with_timeout(
10193        spec,
10194        runtime,
10195        registry,
10196        snapshot,
10197        None,
10198        RouteCloseReason::Restart,
10199        drain_timeout,
10200    )
10201    .await?;
10202
10203    if child.is_some() {
10204        drain_optional_child(
10205            &spec.module_id,
10206            spec.protocol,
10207            stop_notice,
10208            registry,
10209            runtime.forwarding.as_deref(),
10210            snapshot,
10211            &runtime.terminal_ring,
10212            &runtime.spawn_events,
10213            child,
10214            drain_timeout,
10215            ModuleState::Restarting,
10216            Some(true),
10217            &FailureContext::operator(),
10218        )
10219        .await?;
10220    } else {
10221        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10222            state.enabled = true;
10223            state.state = ModuleState::Restarting;
10224            clear_current_process_facts(state);
10225        })?;
10226        release_dead_registration(
10227            registry,
10228            runtime.forwarding.as_deref(),
10229            snapshot,
10230            &spec.module_id,
10231            &runtime.failure_context(false),
10232        )
10233        .await?;
10234    }
10235
10236    if reset_budget {
10237        reset_restart_count(snapshot, &spec.module_id)?;
10238    }
10239    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10240    schedule_respawn(
10241        runtime,
10242        snapshot,
10243        &spec.module_id,
10244        runtime.restart_policy.backoff,
10245        RespawnKind::Spawn,
10246        false,
10247    )
10248}
10249
10250async fn reload_child(
10251    spec: &ModuleSpec,
10252    runtime: &SupervisorRuntimeConfig,
10253    registry: &Registry,
10254    process_liveness: &SupervisorProcessLiveness,
10255    snapshot: &SharedSnapshot,
10256    child: &mut Option<SupervisedChild>,
10257) -> Result<(), SuperviseError> {
10258    // Reload cycles a running module; it must not silently start a disabled one.
10259    if !lock_snapshot(snapshot)?.enabled {
10260        return Err(SuperviseError::Disabled {
10261            module_id: spec.module_id.clone(),
10262        });
10263    }
10264    let stop_notice = begin_forwarding_drain(
10265        spec,
10266        runtime,
10267        registry,
10268        snapshot,
10269        Some(true),
10270        RouteCloseReason::Reload,
10271    )
10272    .await?;
10273
10274    if child.is_some() {
10275        drain_optional_child(
10276            &spec.module_id,
10277            spec.protocol,
10278            stop_notice,
10279            registry,
10280            runtime.forwarding.as_deref(),
10281            snapshot,
10282            &runtime.terminal_ring,
10283            &runtime.spawn_events,
10284            child,
10285            runtime.drain_timeout,
10286            ModuleState::Restarting,
10287            Some(true),
10288            &FailureContext::operator(),
10289        )
10290        .await?;
10291    } else {
10292        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10293            state.enabled = true;
10294            state.state = ModuleState::Restarting;
10295            clear_current_process_facts(state);
10296        })?;
10297        release_dead_registration(
10298            registry,
10299            runtime.forwarding.as_deref(),
10300            snapshot,
10301            &spec.module_id,
10302            &runtime.failure_context(false),
10303        )
10304        .await?;
10305    }
10306
10307    reset_restart_count(snapshot, &spec.module_id)?;
10308    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10309    schedule_respawn(
10310        runtime,
10311        snapshot,
10312        &spec.module_id,
10313        runtime.restart_policy.backoff,
10314        RespawnKind::Reload,
10315        false,
10316    )
10317}
10318
10319async fn finish_reload_child(
10320    spec: &ModuleSpec,
10321    runtime: &SupervisorRuntimeConfig,
10322    registry: &Registry,
10323    process_liveness: &SupervisorProcessLiveness,
10324    snapshot: &SharedSnapshot,
10325    child: &mut Option<SupervisedChild>,
10326) -> Result<(), SuperviseError> {
10327    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10328    let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10329        Ok(next_child) => next_child,
10330        Err(error @ SuperviseError::ConfigDeliveryDeferred { .. }) => {
10331            defer_config_spawn(
10332                spec,
10333                runtime,
10334                snapshot,
10335                RespawnKind::Reload,
10336                &runtime.failure_context(runtime.recovery_is_daemon),
10337            )?;
10338            return Err(error);
10339        }
10340        Err(error @ SuperviseError::ConfigHeld { .. }) => {
10341            let failure = runtime.failure_context(runtime.recovery_is_daemon);
10342            if try_park_config_attempt(spec, runtime, snapshot, &failure, None)? {
10343                return Err(error);
10344            }
10345            defer_config_spawn(spec, runtime, snapshot, RespawnKind::Reload, &failure)?;
10346            return Err(SuperviseError::ConfigDeliveryDeferred {
10347                module_id: spec.module_id.clone(),
10348            });
10349        }
10350        Err(err) => {
10351            return handle_reload_spawn_failure(
10352                spec,
10353                runtime,
10354                process_liveness,
10355                snapshot,
10356                child,
10357                format!("new child failed to spawn: {err}"),
10358            )
10359            .await;
10360        }
10361    };
10362    *child = Some(next_child);
10363
10364    let wait_outcome = {
10365        let active_child = child.as_mut().expect("new reload child was just stored");
10366        wait_for_registration_after_reload(
10367            registry,
10368            &spec.module_id,
10369            snapshot,
10370            active_child,
10371            REGISTRY_RELEASE_TIMEOUT,
10372        )
10373        .await?
10374    };
10375
10376    match wait_outcome {
10377        RegistrationWaitOutcome::Registered => {
10378            child
10379                .as_mut()
10380                .expect("registered reload child")
10381                .registration_deadline = None;
10382            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
10383            Ok(())
10384        }
10385        RegistrationWaitOutcome::Exited(exit_report) => {
10386            if let Some(active_child) = child.as_mut() {
10387                active_child.drain_stderr(&spec.module_id).await;
10388            }
10389            // Keep the exited child's roster guard until its terminal is written.
10390            // Shutdown waits on that guard, not on the child Option used for respawn.
10391            let mut exited_child = child.take().expect("exited reload child is still stored");
10392            #[cfg(test)]
10393            if let Some(gate) = &runtime.test_reload_exit_record_gate {
10394                gate.reached.notify_one();
10395                gate.resume.notified().await;
10396            }
10397            let result = handle_reload_child_registration_failure(
10398                spec,
10399                runtime,
10400                registry,
10401                process_liveness,
10402                snapshot,
10403                child,
10404                ReloadRegistrationFailure {
10405                    exit_report: registration_failure_exit_report(exit_report),
10406                    reason: exited_child
10407                        .spawn_failure
10408                        .clone()
10409                        .unwrap_or_else(|| "new child exited before registering".to_string()),
10410                },
10411            )
10412            .await;
10413            exited_child
10414                .finish_exit(false)
10415                .await
10416                .map_err(|source| SuperviseError::Wait {
10417                    module_id: spec.module_id.clone(),
10418                    source,
10419                })?;
10420            result
10421        }
10422        RegistrationWaitOutcome::TimedOut => {
10423            let mut timed_out_child = child
10424                .take()
10425                .expect("timed-out reload child is still running");
10426            timed_out_child
10427                .start_kill()
10428                .map_err(|source| SuperviseError::Kill {
10429                    module_id: spec.module_id.clone(),
10430                    source,
10431                })?;
10432            let status = timed_out_child
10433                .wait()
10434                .await
10435                .map_err(|source| SuperviseError::Wait {
10436                    module_id: spec.module_id.clone(),
10437                    source,
10438                })?;
10439            timed_out_child.drain_stderr(&spec.module_id).await;
10440            let result = handle_reload_child_registration_failure(
10441                spec,
10442                runtime,
10443                registry,
10444                process_liveness,
10445                snapshot,
10446                child,
10447                ReloadRegistrationFailure {
10448                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
10449                        snapshot,
10450                        &timed_out_child,
10451                        &status,
10452                    )),
10453                    reason: format!(
10454                        "new child did not register within {:?}",
10455                        REGISTRY_RELEASE_TIMEOUT
10456                    ),
10457                },
10458            )
10459            .await;
10460            timed_out_child
10461                .finish_exit(false)
10462                .await
10463                .map_err(|source| SuperviseError::Wait {
10464                    module_id: spec.module_id.clone(),
10465                    source,
10466                })?;
10467            result
10468        }
10469    }
10470}
10471
10472#[allow(clippy::too_many_arguments)]
10473async fn set_child_enabled(
10474    spec: &ModuleSpec,
10475    runtime: &SupervisorRuntimeConfig,
10476    registry: &Registry,
10477    process_liveness: &SupervisorProcessLiveness,
10478    snapshot: &SharedSnapshot,
10479    child: &mut Option<SupervisedChild>,
10480    enabled: bool,
10481    failure: &FailureContext,
10482) -> Result<bool, SuperviseError> {
10483    set_child_enabled_with_budget_policy(
10484        spec,
10485        runtime,
10486        registry,
10487        process_liveness,
10488        snapshot,
10489        child,
10490        enabled,
10491        failure,
10492        true,
10493    )
10494    .await
10495}
10496
10497#[allow(clippy::too_many_arguments)]
10498async fn set_child_enabled_with_budget_policy(
10499    spec: &ModuleSpec,
10500    runtime: &SupervisorRuntimeConfig,
10501    registry: &Registry,
10502    process_liveness: &SupervisorProcessLiveness,
10503    snapshot: &SharedSnapshot,
10504    child: &mut Option<SupervisedChild>,
10505    enabled: bool,
10506    failure: &FailureContext,
10507    reset_budget: bool,
10508) -> Result<bool, SuperviseError> {
10509    let (current_enabled, current_state, respawn_pending) = {
10510        let state = lock_snapshot(snapshot)?;
10511        (state.enabled, state.state, state.respawn_pending)
10512    };
10513    // `start` (enable on an already-enabled module) heals TERMINAL states instead
10514    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
10515    // clean (Stopped) has no live process and no other in-band recovery — the
10516    // operator's start is the explicit recovery act and resets the budget. Without
10517    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
10518    // which the 2026-07-14 aft outage proved is a trap when the failed module is
10519    // the one providing every agent's shell.
10520    let revive_terminal = enabled
10521        && current_enabled
10522        && child.is_none()
10523        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
10524            || (current_state == ModuleState::Restarting && !respawn_pending));
10525    if current_enabled == enabled && !revive_terminal {
10526        return Ok(false);
10527    }
10528
10529    if enabled {
10530        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10531            state.enabled = true;
10532            state.state = ModuleState::Starting;
10533            clear_current_process_facts(state);
10534        })?;
10535        #[cfg(test)]
10536        if runtime.test_seed_stale_facts_before_enable_spawn {
10537            update_snapshot(snapshot, Some(&spec.module_id), |state| {
10538                state.process_alive = true;
10539                state.pid = Some(41);
10540                state.spawned_at_ms = Some(42);
10541                state.spawned_from = Some(PathBuf::from("/spawned/module"));
10542                state.spawned_file_identity = Some(SpawnedFileIdentity {
10543                    device: 43,
10544                    inode: 44,
10545                    #[cfg(windows)]
10546                    windows_image: None,
10547                });
10548            })?;
10549        }
10550        release_dead_registration(
10551            registry,
10552            runtime.forwarding.as_deref(),
10553            snapshot,
10554            &spec.module_id,
10555            failure,
10556        )
10557        .await?;
10558        if reset_budget {
10559            reset_restart_count(snapshot, &spec.module_id)?;
10560        }
10561        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10562        let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10563            Ok(next_child) => next_child,
10564            Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
10565                defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
10566                return Ok(true);
10567            }
10568            Err(error @ SuperviseError::ConfigHeld { .. }) => {
10569                if let Err(parking_error) = hold_config_spawn(spec, runtime, snapshot, failure) {
10570                    process_liveness.untrack_if_current(&spec.module_id, snapshot);
10571                    return Err(parking_error);
10572                }
10573                return Err(error);
10574            }
10575            Err(err) => {
10576                fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
10577                process_liveness.untrack_if_current(&spec.module_id, snapshot);
10578                return Err(err);
10579            }
10580        };
10581        *child = Some(next_child);
10582        debug!(module_id = %spec.module_id, "supervised module enabled");
10583        Ok(true)
10584    } else {
10585        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10586            state.config_hold_parked = false;
10587            state.config_hold_daemon_initiated = false;
10588            state.config_hold_approval = None;
10589        })?;
10590        let stop_notice = begin_forwarding_drain_if_configured(
10591            spec,
10592            runtime,
10593            registry,
10594            snapshot,
10595            Some(false),
10596            RouteCloseReason::Disable,
10597        )
10598        .await?;
10599        drain_optional_child(
10600            &spec.module_id,
10601            spec.protocol,
10602            stop_notice,
10603            registry,
10604            runtime.forwarding.as_deref(),
10605            snapshot,
10606            &runtime.terminal_ring,
10607            &runtime.spawn_events,
10608            child,
10609            runtime.drain_timeout,
10610            ModuleState::Disabled,
10611            Some(false),
10612            failure,
10613        )
10614        .await?;
10615        debug!(module_id = %spec.module_id, "supervised module disabled");
10616        Ok(true)
10617    }
10618}
10619
10620#[allow(clippy::too_many_arguments)]
10621async fn on_child_exit(
10622    spec: &ModuleSpec,
10623    policy: RestartPolicy,
10624    registry: &Registry,
10625    snapshot: &SharedSnapshot,
10626    terminal_ring: &Arc<Mutex<TerminalRing>>,
10627    spawn_events: &SpawnEventFeed,
10628    roster: &ChildRoster,
10629    exit_report: ExitReport,
10630    daemon_initiated: bool,
10631) -> NextAction {
10632    // Once the daemon has begun shutting down, no exit is a crash to recover
10633    // from: the module is exiting because the daemon is going away (EOF on its
10634    // connection, or a service manager signalling the whole cgroup). Record it
10635    // as such and never schedule a respawn, which would only start a process
10636    // for the shutdown to end again.
10637    if roster.is_closed() {
10638        return on_child_exit_during_daemon_shutdown(
10639            spec,
10640            registry,
10641            snapshot,
10642            terminal_ring,
10643            spawn_events,
10644            exit_report,
10645        )
10646        .await;
10647    }
10648    // Deliberate stops the supervisor asks for (operator stop, disable,
10649    // restart, reload, swap, a health restart, a drain that runs out of budget)
10650    // takes the child out of the supervise loop and reaps it in
10651    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
10652    // that reaches this point is either unrequested or a first-HELLO timeout,
10653    // which is deliberately classified as a crash even after a clean TERM exit.
10654    //
10655    // For a subc-wire module a clean exit is still a stop: those modules are
10656    // written to re-raise SIGTERM, so a stray outside signal already reads as a
10657    // crash, and exiting 0 is a deliberate choice the module made. A
10658    // `protocol: "none"` module is a stock program we cannot change, and many
10659    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
10660    // stop would leave the module down for good after any stray signal, so it
10661    // goes through the crash path instead: it spends restart budget, respawns
10662    // with the crash backoff, and ends `failed` when the budget runs out.
10663    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
10664        && running_protocol(spec, snapshot) == ModuleProtocol::None;
10665    match exit_report.kind {
10666        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
10667            info!(
10668                module_id = %spec.module_id,
10669                exit_code = ?exit_report.code,
10670                exit_signal = ?exit_report.signal,
10671                "supervised module exited cleanly"
10672            );
10673            if let Err(err) = publish_after_terminal_record(
10674                &spec.module_id,
10675                || {
10676                    record_terminal(
10677                        &spec.module_id,
10678                        terminal_ring,
10679                        spawn_events,
10680                        &exit_report,
10681                        TerminalDisposition::Stopped,
10682                    )
10683                },
10684                || {
10685                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10686                        state.state = ModuleState::Stopped;
10687                        clear_current_process_facts(state);
10688                        state.last_exit = Some(exit_report.clone());
10689                    })
10690                },
10691            ) {
10692                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
10693            }
10694            let registration_released = match wait_for_registration_release(
10695                registry,
10696                &spec.module_id,
10697                REGISTRY_RELEASE_TIMEOUT,
10698            )
10699            .await
10700            {
10701                Ok(()) => true,
10702                Err(err) => {
10703                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
10704                    false
10705                }
10706            };
10707            NextAction::Stop {
10708                registration_released,
10709            }
10710        }
10711        ExitKind::Clean | ExitKind::Crash => {
10712            if unrequested_clean_exit_of_protocol_none {
10713                warn!(
10714                    module_id = %spec.module_id,
10715                    exit_code = ?exit_report.code,
10716                    exit_signal = ?exit_report.signal,
10717                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
10718                );
10719            } else {
10720                warn!(
10721                    module_id = %spec.module_id,
10722                    exit_code = ?exit_report.code,
10723                    exit_signal = ?exit_report.signal,
10724                    "supervised module exited abnormally (crash)"
10725                );
10726            }
10727            let mut restart_schedule = None;
10728            let mut disposition = TerminalDisposition::Disabled;
10729            let mut final_state = ModuleState::Disabled;
10730            // Set only when the budget is what stopped the module, so the
10731            // terminal record says which limit was hit rather than leaving
10732            // `failed` to be read as "crashed once, badly".
10733            let mut disposition_detail = lock_snapshot(snapshot)
10734                .ok()
10735                .and_then(|mut state| state.spawn_failure.take());
10736            let now = Instant::now();
10737            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10738                if state.enabled {
10739                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
10740                        // Reserving the last slot exhausts the budget for the
10741                        // replacement, not for this exit. Publish its scheduled
10742                        // recovery atomically with the reservation so connection
10743                        // cleanup sees it while terminal recording still holds
10744                        // the visible state at Running.
10745                        state.respawn_pending = true;
10746                        final_state = ModuleState::Restarting;
10747                        restart_schedule = Some(schedule);
10748                        disposition = TerminalDisposition::Restarting;
10749                    } else {
10750                        final_state = ModuleState::Failed;
10751                        disposition = TerminalDisposition::Failed;
10752                        let budget = policy.budget_exhausted_detail();
10753                        disposition_detail =
10754                            Some(disposition_detail.take().map_or_else(
10755                                || budget.clone(),
10756                                |cause| format!("{cause}; {budget}"),
10757                            ));
10758                    }
10759                } else {
10760                    disposition = TerminalDisposition::Disabled;
10761                }
10762            }) {
10763                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
10764                return NextAction::Stop {
10765                    registration_released: false,
10766                };
10767            }
10768            if disposition == TerminalDisposition::Failed {
10769                // The window is in the message, not only in the fields: this line
10770                // is read in a scrollback where a bare `max_restarts=3` reads as a
10771                // lifetime cap and sends the operator looking for three crashes
10772                // that never happened together.
10773                error!(
10774                    module_id = %spec.module_id,
10775                    max_restarts = policy.max_restarts,
10776                    window_secs = policy.window.as_secs(),
10777                    "module stopped: {}",
10778                    policy.budget_exhausted_detail()
10779                );
10780            }
10781            if let Err(err) = publish_after_terminal_record(
10782                &spec.module_id,
10783                || {
10784                    record_terminal_with_detail(
10785                        &spec.module_id,
10786                        terminal_ring,
10787                        spawn_events,
10788                        &exit_report,
10789                        disposition,
10790                        disposition_detail,
10791                    )
10792                },
10793                || {
10794                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10795                        state.state = final_state;
10796                        clear_current_process_facts(state);
10797                        if final_state == ModuleState::Failed
10798                            && daemon_initiated
10799                            && !roster.is_closed()
10800                        {
10801                            state.failed_retry.schedule(&spec.module_id);
10802                        }
10803                        state.last_exit = Some(exit_report.clone());
10804                    })
10805                },
10806            ) {
10807                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
10808            }
10809
10810            if let Some(schedule) = restart_schedule {
10811                NextAction::Restart {
10812                    schedule: Some(schedule),
10813                }
10814            } else {
10815                let registration_released = match wait_for_registration_release(
10816                    registry,
10817                    &spec.module_id,
10818                    REGISTRY_RELEASE_TIMEOUT,
10819                )
10820                .await
10821                {
10822                    Ok(()) => true,
10823                    Err(err) => {
10824                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
10825                        false
10826                    }
10827                };
10828                NextAction::Stop {
10829                    registration_released,
10830                }
10831            }
10832        }
10833        ExitKind::DeliberateSeverance => {
10834            warn!(
10835                module_id = %spec.module_id,
10836                exit_code = ?exit_report.code,
10837                exit_signal = ?exit_report.signal,
10838                "supervised module exited after deliberate connection severance"
10839            );
10840            let should_restart = match lock_snapshot(snapshot) {
10841                Ok(state) => state.enabled,
10842                Err(err) => {
10843                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
10844                    return NextAction::Stop {
10845                        registration_released: false,
10846                    };
10847                }
10848            };
10849            let final_state = if should_restart {
10850                ModuleState::Restarting
10851            } else {
10852                ModuleState::Disabled
10853            };
10854            if let Err(err) = publish_after_terminal_record(
10855                &spec.module_id,
10856                || {
10857                    record_terminal(
10858                        &spec.module_id,
10859                        terminal_ring,
10860                        spawn_events,
10861                        &exit_report,
10862                        terminal_disposition(final_state),
10863                    )
10864                },
10865                || {
10866                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10867                        state.state = final_state;
10868                        clear_current_process_facts(state);
10869                        state.last_exit = Some(exit_report.clone());
10870                        state.lifetime_restarts += 1;
10871                    })
10872                },
10873            ) {
10874                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
10875                return NextAction::Stop {
10876                    registration_released: false,
10877                };
10878            }
10879
10880            if should_restart {
10881                NextAction::Restart { schedule: None }
10882            } else {
10883                let registration_released = match wait_for_registration_release(
10884                    registry,
10885                    &spec.module_id,
10886                    REGISTRY_RELEASE_TIMEOUT,
10887                )
10888                .await
10889                {
10890                    Ok(()) => true,
10891                    Err(err) => {
10892                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
10893                        false
10894                    }
10895                };
10896                NextAction::Stop {
10897                    registration_released,
10898                }
10899            }
10900        }
10901    }
10902}
10903
10904async fn on_child_exit_during_daemon_shutdown(
10905    spec: &ModuleSpec,
10906    registry: &Registry,
10907    snapshot: &SharedSnapshot,
10908    terminal_ring: &Arc<Mutex<TerminalRing>>,
10909    spawn_events: &SpawnEventFeed,
10910    exit_report: ExitReport,
10911) -> NextAction {
10912    info!(
10913        module_id = %spec.module_id,
10914        exit_code = ?exit_report.code,
10915        exit_signal = ?exit_report.signal,
10916        exit_kind = ?exit_report.kind,
10917        "supervised module exited during daemon shutdown; not restarting it"
10918    );
10919    if let Err(err) = publish_after_terminal_record(
10920        &spec.module_id,
10921        || {
10922            record_terminal(
10923                &spec.module_id,
10924                terminal_ring,
10925                spawn_events,
10926                &exit_report,
10927                TerminalDisposition::DaemonShutdown,
10928            )
10929        },
10930        || {
10931            update_snapshot(snapshot, Some(&spec.module_id), |state| {
10932                state.state = ModuleState::Stopped;
10933                clear_current_process_facts(state);
10934                state.last_exit = Some(exit_report.clone());
10935            })
10936        },
10937    ) {
10938        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
10939    }
10940    let registration_released =
10941        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
10942            .await
10943            .is_ok();
10944    NextAction::Stop {
10945        registration_released,
10946    }
10947}
10948
10949// Keep the status lock out of terminal I/O. Readers may observe the previous
10950// state while recording is in progress, but never a completed exit without its
10951// history. Journal errors are logged by the journal and retained in the ring;
10952// a recording panic still publishes the final state before resuming the unwind.
10953fn publish_after_terminal_record<T>(
10954    module_id: &str,
10955    record: impl FnOnce(),
10956    publish: impl FnOnce() -> T,
10957) -> T {
10958    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
10959    let published = publish();
10960    if let Err(panic) = recorded {
10961        error!(
10962            module_id,
10963            "terminal recording panicked; published final state before resuming unwind"
10964        );
10965        std::panic::resume_unwind(panic);
10966    }
10967    published
10968}
10969
10970fn fail_after_wait_error(
10971    module_id: &str,
10972    snapshot: &SharedSnapshot,
10973    terminal_ring: &Arc<Mutex<TerminalRing>>,
10974    spawn_events: &SpawnEventFeed,
10975    detail: String,
10976    failure: &FailureContext,
10977) {
10978    let report = wait_error_exit_report();
10979    publish_after_terminal_record(
10980        module_id,
10981        || {
10982            record_terminal_with_detail(
10983                module_id,
10984                terminal_ring,
10985                spawn_events,
10986                &report,
10987                TerminalDisposition::Failed,
10988                Some(detail),
10989            )
10990        },
10991        || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
10992    );
10993}
10994
10995#[cfg(test)]
10996fn record_wait_error_terminal(
10997    module_id: &str,
10998    terminal_ring: &Arc<Mutex<TerminalRing>>,
10999    spawn_events: &SpawnEventFeed,
11000) {
11001    record_terminal(
11002        module_id,
11003        terminal_ring,
11004        spawn_events,
11005        &wait_error_exit_report(),
11006        TerminalDisposition::Failed,
11007    );
11008}
11009
11010fn record_terminal(
11011    module_id: &str,
11012    terminal_ring: &Arc<Mutex<TerminalRing>>,
11013    spawn_events: &SpawnEventFeed,
11014    exit_report: &ExitReport,
11015    disposition: TerminalDisposition,
11016) {
11017    record_terminal_with_detail(
11018        module_id,
11019        terminal_ring,
11020        spawn_events,
11021        exit_report,
11022        disposition,
11023        None,
11024    );
11025}
11026
11027/// The ring lock is held only to capture the read (see
11028/// `TerminalJournal::capture_read`), so this module's exits keep recording
11029/// while the journal files are read. Blocking: it reads files.
11030fn durable_terminal_history_of(
11031    terminal_ring: &Mutex<TerminalRing>,
11032    module_id: &str,
11033) -> subc_control::TerminalHistory {
11034    let read = terminal_ring
11035        .lock()
11036        .unwrap_or_else(|p| p.into_inner())
11037        .capture_durable_history();
11038    read.read(module_id)
11039}
11040
11041fn record_terminal_with_detail(
11042    module_id: &str,
11043    terminal_ring: &Arc<Mutex<TerminalRing>>,
11044    spawn_events: &SpawnEventFeed,
11045    exit_report: &ExitReport,
11046    disposition: TerminalDisposition,
11047    disposition_detail: Option<String>,
11048) {
11049    #[cfg(test)]
11050    terminal_history_tests::before_terminal_record(spawn_events);
11051    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
11052    let record = TerminalRecord {
11053        exit_code: exit_report.code,
11054        exit_signal: exit_report.signal,
11055        at_ms: exit_report.at_ms,
11056        disposition,
11057        exit_kind: exit_report.kind.into(),
11058        disposition_detail,
11059    };
11060    terminal_ring
11061        .lock()
11062        .unwrap_or_else(|poisoned| poisoned.into_inner())
11063        .record_exit(module_id, record);
11064}
11065
11066fn untrack_if_registration_released(
11067    process_liveness: &SupervisorProcessLiveness,
11068    registry: &Registry,
11069    module_id: &str,
11070    snapshot: &SharedSnapshot,
11071) {
11072    match registry.get_module(module_id) {
11073        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
11074        Ok(Some(_)) => {}
11075        Err(err) => {
11076            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
11077        }
11078    }
11079}
11080
11081/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
11082/// then apply the module's configured entries minus daemon-private capture keys.
11083///
11084/// Separated from `spawn_child` only so it can be asserted without spawning a
11085/// process — a duplicate of this logic in a test would pass while the real one
11086/// drifted, which is the defect class this function exists to avoid.
11087/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
11088/// nonce. A `protocol: "none"` module gets neither, because it cannot use
11089/// either and the argument would stop a stock binary from starting at all.
11090/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
11091///
11092/// The plain-spawn form, kept for the tests that assert its plan; spawns go
11093/// through [`apply_wire_spawn_args_for_admission`].
11094#[cfg(test)]
11095fn apply_wire_spawn_args(
11096    command: &mut Command,
11097    spec: &ModuleSpec,
11098    connection_file_path: Option<&std::path::Path>,
11099    handle: Option<&SupervisorHandle>,
11100) -> Result<Option<NonceHandoff>, SuperviseError> {
11101    apply_wire_spawn_args_for_role(
11102        command,
11103        spec,
11104        connection_file_path,
11105        handle,
11106        SpawnRole::Plain,
11107    )
11108}
11109
11110/// The read end of a spawn's launch-nonce pipe, prepared by
11111/// [`apply_wire_spawn_args_for_admission`] and installed as the child's descriptor 3
11112/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
11113/// server before spawn and authenticates its client after the child exists.
11114#[cfg(unix)]
11115type NonceHandoff = subc_os::LaunchNonceHandoff;
11116#[cfg(windows)]
11117type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
11118#[cfg(not(any(unix, windows)))]
11119type NonceHandoff = std::convert::Infallible;
11120
11121/// Create the launch nonce that a new module process must present when it
11122/// registers or opens a consumer connection, including a replacement process
11123/// started alongside a still-serving module.
11124///
11125/// A plain spawn replaces the module's recorded nonce. A swap candidate records
11126/// a separate candidate token so the still-serving incumbent and its consumers
11127/// keep their nonce. Both records are installed before the process exists, so
11128/// the child's initial HELLO registration cannot arrive ahead of its nonce.
11129///
11130/// On Unix the nonce is delivered only through a pipe. It is written into
11131/// a pipe whose read end the child gets as descriptor 3, named by
11132/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
11133/// process of the same user cannot read it with `ps eww`. That handoff is
11134/// returned rather than installed here. The combined descriptor placement runs
11135/// after the cgroup callback and placement barrier, so replacing slots 3-5
11136/// cannot close a descriptor either callback still needs.
11137/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
11138#[cfg(test)]
11139fn apply_wire_spawn_args_for_role(
11140    command: &mut Command,
11141    spec: &ModuleSpec,
11142    connection_file_path: Option<&std::path::Path>,
11143    handle: Option<&SupervisorHandle>,
11144    role: SpawnRole,
11145) -> Result<Option<NonceHandoff>, SuperviseError> {
11146    apply_wire_spawn_args_for_admission(command, spec, connection_file_path, handle, role, None)
11147}
11148
11149fn apply_wire_spawn_args_for_admission(
11150    command: &mut Command,
11151    spec: &ModuleSpec,
11152    connection_file_path: Option<&std::path::Path>,
11153    handle: Option<&SupervisorHandle>,
11154    role: SpawnRole,
11155    admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11156) -> Result<Option<NonceHandoff>, SuperviseError> {
11157    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
11158    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
11159    // included: a daemon started from a module's process tree inherits it,
11160    // and passing it on would point the child at a descriptor it does not
11161    // have.
11162    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
11163    #[cfg(windows)]
11164    {
11165        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
11166        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
11167    }
11168    // Remove inherited or configured copies too: withholding must mean absent.
11169    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
11170    command.env_remove(subc_os::registration_channel::REGISTRATION_FD_ENV);
11171    command.env_remove(subc_os::registration_channel::REGISTRATION_PIPE_ENV);
11172    command.env_remove(subc_os::registration_channel::DAEMON_PID_ENV);
11173    if role == SpawnRole::Plain {
11174        if let Some(handle) = handle {
11175            #[cfg(all(test, unix))]
11176            {
11177                let pause = {
11178                    let mut slot = handle.plain_identity_policy_pause.lock().unwrap();
11179                    if slot
11180                        .as_ref()
11181                        .is_some_and(|(module, _)| module == &spec.module_id)
11182                    {
11183                        slot.take().map(|(_, pause)| pause)
11184                    } else {
11185                        None
11186                    }
11187                };
11188                if let Some(pause) = pause {
11189                    pause.reached.send(()).unwrap();
11190                    pause.release.recv().unwrap();
11191                }
11192            }
11193            // A rescan preserves the running child's nonce and reservation.
11194            // Replacement spawn preparation applies the new protocol now,
11195            // before protocol:none can return without minting a nonce.
11196            handle.apply_identity_policy(spec, spec.protocol);
11197        }
11198    }
11199    if spec.protocol == ModuleProtocol::None {
11200        return Ok(None);
11201    }
11202    if let Some(connection_file_path) = connection_file_path {
11203        command.arg(SUBC_ARG).arg(connection_file_path);
11204    }
11205
11206    if admission.is_some_and(|admission| admission.launch.is_some()) {
11207        if let Some(handle) = handle {
11208            match role {
11209                SpawnRole::Plain => {
11210                    handle
11211                        .spawn_nonces
11212                        .lock()
11213                        .unwrap_or_else(|p| p.into_inner())
11214                        .remove(&spec.module_id);
11215                    if spec.reserved {
11216                        handle
11217                            .reserved_nonces
11218                            .lock()
11219                            .unwrap_or_else(|p| p.into_inner())
11220                            .insert(spec.module_id.clone(), None);
11221                    }
11222                }
11223                // Swaps still need their lifecycle record, but the empty value
11224                // is not a bearer token and is never accepted by nonce HELLO.
11225                SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, String::new()),
11226            }
11227        }
11228        return Ok(None);
11229    }
11230    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
11231    // route.open attestation. Reserved modules additionally use the same nonce
11232    // for HELLO id-squatting protection. A respawn rotates both records.
11233    let nonce = generate_launch_nonce()?;
11234    if let Some(handle) = handle {
11235        match role {
11236            SpawnRole::Plain => {
11237                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
11238                if spec.reserved {
11239                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
11240                }
11241            }
11242            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
11243        }
11244    }
11245    #[cfg(unix)]
11246    let handoff = {
11247        let handoff =
11248            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
11249                program: spec.program.clone(),
11250                source,
11251                cgroup_path: None,
11252            })?;
11253        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
11254        Some(handoff)
11255    };
11256    #[cfg(windows)]
11257    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
11258    #[cfg(not(any(unix, windows)))]
11259    let handoff = None;
11260    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
11261    // until the daemon's per-spawn delivery records show that every running
11262    // Windows module has read its secret from the named pipe.
11263    #[cfg(not(unix))]
11264    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
11265    Ok(handoff)
11266}
11267
11268pub(crate) fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
11269    command.env_remove(CK_LOG_ENV);
11270    // The spawn role is the supervisor's to set, and only on a swap candidate
11271    // (see `apply_spawn_role`). Removing it here, rather than just not setting
11272    // it, is what makes it absent on a plain spawn: the daemon's own
11273    // environment could carry it, and so could a spec built outside daemon
11274    // config (config refuses it as an `env` key). A module reading it on a
11275    // plain restart would pick the long swap budget and leave callers waiting.
11276    command.env_remove(SUBC_SPAWN_ROLE_ENV);
11277    for (key, value) in &spec.env {
11278        // cortexkit-log currently exposes retention only as a Rust struct, not
11279        // environment names. These values are daemon-private sink metadata and
11280        // must never become a public child-process contract by being inherited.
11281        if matches!(
11282            key.as_str(),
11283            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
11284        ) || key == SUBC_SPAWN_ROLE_ENV
11285        {
11286            continue;
11287        }
11288        command.env(key, value);
11289    }
11290}
11291
11292#[cfg(windows)]
11293fn prepare_windows_registration_pipe(
11294    command: &mut Command,
11295    spec: &ModuleSpec,
11296) -> Result<Option<SpawnRegistrationChannel>, SuperviseError> {
11297    if spec.protocol == ModuleProtocol::None {
11298        return Ok(None);
11299    }
11300    let pipe = subc_os::registration_channel::RegistrationPipe::new().map_err(|source| {
11301        SuperviseError::Spawn {
11302            program: spec.program.clone(),
11303            source,
11304            cgroup_path: None,
11305        }
11306    })?;
11307    command
11308        .env(
11309            subc_os::registration_channel::REGISTRATION_PIPE_ENV,
11310            pipe.name(),
11311        )
11312        .env(
11313            subc_os::registration_channel::DAEMON_PID_ENV,
11314            std::process::id().to_string(),
11315        );
11316    Ok(Some(pipe))
11317}
11318
11319#[cfg(windows)]
11320fn prepare_windows_nonce_pipe(
11321    command: &mut Command,
11322    spec: &ModuleSpec,
11323    nonce: &str,
11324) -> Option<NonceHandoff> {
11325    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
11326        Ok(handoff) => {
11327            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
11328            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
11329            // still register until every module reads the pipe. A reader without
11330            // this variable set to env returns the pipe error instead of using
11331            // an inherited or configured environment copy.
11332            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
11333            Some(handoff)
11334        }
11335        Err(error) => {
11336            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
11337            None
11338        }
11339    }
11340}
11341
11342#[cfg(windows)]
11343fn start_windows_nonce_pipe(
11344    handoff: Option<NonceHandoff>,
11345    pid: u32,
11346    spec: &ModuleSpec,
11347) -> Option<subc_os::LaunchNoncePipeDelivery> {
11348    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
11349    // same maximum wait used for registration during a reload. Older modules
11350    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
11351    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
11352        Ok(delivery) => Some(delivery),
11353        Err(error) => {
11354            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
11355            None
11356        }
11357    })
11358}
11359
11360fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
11361    #[cfg(windows)]
11362    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
11363        return Some(
11364            if snapshot
11365                .pipe_consumed
11366                .as_ref()
11367                .is_some_and(|consumed| consumed.get().is_some())
11368            {
11369                "pipe"
11370            } else {
11371                "env"
11372            }
11373            .to_owned(),
11374        );
11375    }
11376    let _ = snapshot;
11377    None
11378}
11379
11380/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
11381/// of a blue/green swap.
11382#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11383enum SpawnRole {
11384    Plain,
11385    SwapCandidate,
11386}
11387
11388/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
11389/// `apply_child_env` has already removed the variable for every spawn.
11390fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
11391    if role == SpawnRole::SwapCandidate {
11392        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
11393    }
11394}
11395
11396fn previous_instance_error(
11397    spec: &ModuleSpec,
11398    pid: u32,
11399    reason: crate::live_children::RetainedReason,
11400) -> SuperviseError {
11401    SuperviseError::Spawn {
11402        program: spec.program.clone(),
11403        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
11404        cgroup_path: None,
11405    }
11406}
11407
11408fn carries_containment(configured: bool, has_container: bool) -> bool {
11409    configured || has_container
11410}
11411
11412fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
11413    if !carries_containment(
11414        spec.carry_containment,
11415        containment::current(&roster.containment).is_some(),
11416    ) {
11417        return None;
11418    }
11419    Some(SuperviseError::Spawn {
11420        program: spec.program.clone(),
11421        source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
11422        cgroup_path: None,
11423    })
11424}
11425
11426#[allow(clippy::too_many_arguments)]
11427fn spawn_child(
11428    spec: &ModuleSpec,
11429    connection_file_path: Option<&std::path::Path>,
11430    handle: Option<&SupervisorHandle>,
11431    ring: &Arc<Mutex<StderrRing>>,
11432    capture_logs_dir: Option<&std::path::Path>,
11433    roster: &ChildRoster,
11434    config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11435    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11436    register_timeout: Duration,
11437) -> Result<SupervisedChild, SuperviseError> {
11438    #[cfg(all(test, unix))]
11439    if let Some(source) = containment_tests::join_failure(spec, roster) {
11440        containment::terminate_blocking(&roster.containment);
11441        return Err(SuperviseError::Spawn {
11442            program: spec.program.clone(),
11443            source,
11444            cgroup_path: None,
11445        });
11446    }
11447    let result = spawn_child_in_slot(
11448        spec,
11449        connection_file_path,
11450        handle,
11451        ring,
11452        capture_logs_dir,
11453        roster,
11454        config_delivery,
11455        #[cfg(target_os = "linux")]
11456        cgroup_placement,
11457        SpawnRole::Plain,
11458        false,
11459        register_timeout,
11460    );
11461    if result.is_err() {
11462        if roster.admission_pending(&spec.module_id) {
11463            containment::terminate_deferred(&roster.containment);
11464        } else {
11465            containment::terminate_blocking(&roster.containment);
11466        }
11467    }
11468    result
11469}
11470
11471/// Spawn one process of `spec` into a slot.
11472///
11473/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
11474/// A swap candidate needs a different cgroup from the process it is replacing,
11475/// which is still alive: in the same cgroup the two would be one kill domain,
11476/// and killing a failed candidate could take the incumbent with it.
11477///
11478/// The stderr capture file is `<module_id>.stderr.log` for every process of
11479/// the module, whichever slot it is in, because that is the one file
11480/// `ck module logs` reads. During a swap's overlap both processes append to it;
11481/// the daemon writes whole lines, so the two interleave by line, which is also
11482/// the merged view an operator wants while a swap runs.
11483#[allow(clippy::too_many_arguments)]
11484fn spawn_child_in_slot(
11485    spec: &ModuleSpec,
11486    connection_file_path: Option<&std::path::Path>,
11487    handle: Option<&SupervisorHandle>,
11488    ring: &Arc<Mutex<StderrRing>>,
11489    capture_logs_dir: Option<&std::path::Path>,
11490    roster: &ChildRoster,
11491    config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11492    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11493    role: SpawnRole,
11494    alternate_slot: bool,
11495    register_timeout: Duration,
11496) -> Result<SupervisedChild, SuperviseError> {
11497    let prepared = prepare_child_in_slot(
11498        spec,
11499        connection_file_path,
11500        handle,
11501        capture_logs_dir,
11502        roster,
11503        config_delivery,
11504        #[cfg(target_os = "linux")]
11505        cgroup_placement,
11506        role,
11507        alternate_slot,
11508        false,
11509    )?;
11510    let gate = config_delivery
11511        .map(|delivery| delivery.module_gate(spec))
11512        .transpose()?;
11513    let _gate = gate
11514        .as_ref()
11515        .map(|gate| {
11516            gate.try_lock()
11517                .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11518                    module_id: spec.module_id.clone(),
11519                })
11520        })
11521        .transpose()?;
11522    if let (Some(delivery), Some(admission)) = (config_delivery, &prepared.spawn_admission) {
11523        delivery.check(spec, admission)?;
11524    }
11525    let checked_base = prepared
11526        .spawn_admission
11527        .as_ref()
11528        .and_then(|admission| admission.approved_base())
11529        .cloned();
11530    let mut result = launch_child_in_slot(spec, ring, roster, prepared, register_timeout);
11531    drop(_gate);
11532    if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11533        (config_delivery, &result)
11534    {
11535        if let Some(base) = checked_base.as_ref().filter(|_| code == "program_refused") {
11536            if !delivery.record_refusal(spec, base, detail)? {
11537                result = Err(spawn_plan_stale(spec));
11538            }
11539        }
11540    }
11541    result
11542}
11543
11544struct AsyncInitialIdentity<'a> {
11545    supervisor: &'a Supervisor,
11546    module: String,
11547    committed: bool,
11548}
11549impl Drop for AsyncInitialIdentity<'_> {
11550    fn drop(&mut self) {
11551        if !self.committed {
11552            self.supervisor.abandon_unrostered(&self.module);
11553        }
11554    }
11555}
11556
11557/// Child-free preparation can be abandoned without orphaning a process. After
11558/// this value returns, the async caller validates its launch witness and creates
11559/// and publishes the child without another await.
11560struct PreparedChild {
11561    script_delivery: Option<script::Delivery>,
11562    registration_spawn: Option<Arc<RegistrationSpawn>>,
11563    registration_handle: Option<SupervisorHandle>,
11564    #[cfg(any(unix, windows))]
11565    registration_channel: Option<SpawnRegistrationChannel>,
11566    config_context: Option<config_delivery::ConfigDelivery>,
11567    binding_publication: Option<ProgramBindingPublication>,
11568    launch_spec: ModuleSpec,
11569    spawn_admission: Option<crate::config_gate::runtime::SpawnAdmission>,
11570    command: Command,
11571    carrying: bool,
11572    carried: Option<Arc<containment::CarriedContainment>>,
11573    #[cfg(unix)]
11574    launch_identity: Option<containment::LaunchIdentity>,
11575    #[cfg(unix)]
11576    group_target: containment::GroupTarget,
11577    #[cfg(target_os = "linux")]
11578    cgroup_placement: Option<subc_cgroup::Placement>,
11579    #[cfg(target_os = "linux")]
11580    cgroup_name: String,
11581    #[cfg(target_os = "linux")]
11582    cgroup_cleanup: PreparedCgroup,
11583    cgroup_path: Option<PathBuf>,
11584    #[cfg(unix)]
11585    admission_gate: Option<admission::Gate>,
11586    output_sink: ChildOutputSink,
11587    #[cfg(target_os = "macos")]
11588    privacy_exec: Option<PrivacyExec>,
11589    #[cfg(windows)]
11590    image_capture: subc_os::ExecutableCapture,
11591    #[cfg(windows)]
11592    nonce_handoff: Option<NonceHandoff>,
11593    spawned_at: Instant,
11594    config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
11595    delivery_registration: Option<config_delivery::RegistrationGuard>,
11596    program_binding: Option<SpawnProgramBinding>,
11597}
11598
11599#[cfg(target_os = "linux")]
11600struct PreparedCgroup(Option<(subc_cgroup::Placement, String)>);
11601#[cfg(target_os = "linux")]
11602impl Drop for PreparedCgroup {
11603    fn drop(&mut self) {
11604        if let Some((placement, name)) = &self.0 {
11605            remove_module_cgroup(placement, name);
11606        }
11607    }
11608}
11609
11610pub(crate) struct SpawnProgramBinding {
11611    pub(crate) path: PathBuf,
11612    // Keep the inode alive if the configured name is replaced while preparation
11613    // is suspended, so replacement cannot pass through inode reuse.
11614    _file: std::fs::File,
11615    pub(crate) identity: subc_os::RetainedFileIdentity,
11616}
11617
11618impl SpawnProgramBinding {
11619    pub(crate) fn capture(spec: &ModuleSpec) -> Result<Self, SuperviseError> {
11620        Self::capture_after_resolution(spec, |_| {})
11621    }
11622
11623    fn capture_after_resolution(
11624        spec: &ModuleSpec,
11625        before_open: impl FnOnce(&std::path::Path),
11626    ) -> Result<Self, SuperviseError> {
11627        require_program_search_path(spec)?;
11628        #[cfg(windows)]
11629        let windows_capture =
11630            windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
11631                program: spec.program.clone(),
11632                source,
11633                cgroup_path: None,
11634            })?;
11635        #[cfg(windows)]
11636        let path = windows_capture.path().to_path_buf();
11637        #[cfg(not(windows))]
11638        let path = if spec.program.is_absolute() || spec.program.components().count() > 1 {
11639            spec.program.clone()
11640        } else {
11641            let path = spec
11642                .env
11643                .iter()
11644                .rev()
11645                .find(|(key, _)| key == "PATH")
11646                .map(|(_, path)| std::ffi::OsString::from(path))
11647                .or_else(|| std::env::var_os("PATH"))
11648                .unwrap_or_else(|| {
11649                    if cfg!(target_os = "macos") {
11650                        // Match Darwin's spawn search when PATH is absent, not
11651                        // the current-directory search of an explicitly empty PATH.
11652                        "/usr/bin:/bin".into()
11653                    } else {
11654                        Default::default()
11655                    }
11656                });
11657            std::env::split_paths(&path)
11658                .map(|dir| dir.join(&spec.program))
11659                .find(|path| {
11660                    if !path.is_file() {
11661                        return false;
11662                    }
11663                    #[cfg(unix)]
11664                    {
11665                        rustix::fs::accessat(
11666                            rustix::fs::CWD,
11667                            path,
11668                            rustix::fs::Access::EXEC_OK,
11669                            rustix::fs::AtFlags::EACCESS,
11670                        )
11671                        .is_ok()
11672                    }
11673                    #[cfg(not(unix))]
11674                    {
11675                        true
11676                    }
11677                })
11678                .ok_or_else(|| SuperviseError::Spawn {
11679                    program: spec.program.clone(),
11680                    source: io::Error::new(io::ErrorKind::NotFound, "executable not found in PATH"),
11681                    cgroup_path: None,
11682                })?
11683        };
11684        let resolved = std::fs::canonicalize(&path).map_err(|source| SuperviseError::Spawn {
11685            program: spec.program.clone(),
11686            source,
11687            cgroup_path: None,
11688        })?;
11689        before_open(&resolved);
11690        // Resolving links does not open their target. The descriptor open must
11691        // remain nonblocking and no-follow if that resolved name is substituted.
11692        let file =
11693            subc_os::open_regular_nofollow(&resolved).map_err(|source| SuperviseError::Spawn {
11694                program: spec.program.clone(),
11695                source: io::Error::other(source),
11696                cgroup_path: None,
11697            })?;
11698        let identity =
11699            subc_os::retained_file_identity(&file).map_err(|source| SuperviseError::Spawn {
11700                program: spec.program.clone(),
11701                source,
11702                cgroup_path: None,
11703            })?;
11704        Ok(Self {
11705            path,
11706            _file: file,
11707            identity,
11708        })
11709    }
11710    fn current(&self) -> bool {
11711        #[cfg(unix)]
11712        {
11713            subc_os::file_identity(&self.path) == Some(self.identity)
11714        }
11715        #[cfg(windows)]
11716        {
11717            std::fs::canonicalize(&self.path)
11718                .ok()
11719                .and_then(|path| subc_os::open_regular_nofollow(&path).ok())
11720                .and_then(|file| subc_os::retained_file_identity(&file).ok())
11721                == Some(self.identity)
11722        }
11723    }
11724}
11725
11726fn spawn_plan_stale(spec: &ModuleSpec) -> SuperviseError {
11727    SuperviseError::ConfigDeliveryDeferred {
11728        module_id: spec.module_id.clone(),
11729    }
11730}
11731
11732fn require_program_search_path(spec: &ModuleSpec) -> Result<(), SuperviseError> {
11733    if !cfg!(target_os = "macos")
11734        && !spec.program.is_absolute()
11735        && spec.program.components().count() == 1
11736        && !spec.env.iter().any(|(key, _)| {
11737            if cfg!(windows) {
11738                key.eq_ignore_ascii_case("PATH")
11739            } else {
11740                key == "PATH"
11741            }
11742        })
11743        && std::env::var_os("PATH").is_none()
11744    {
11745        // The OS default search is not a current-directory search. Refusing an
11746        // unspecified search path prevents approving a decoy in the current directory.
11747        return Err(SuperviseError::Spawn {
11748            program: spec.program.clone(),
11749            source: io::Error::new(io::ErrorKind::InvalidInput, "bare program requires PATH"),
11750            cgroup_path: None,
11751        });
11752    }
11753    Ok(())
11754}
11755
11756async fn spawn_child_async(
11757    spec: &ModuleSpec,
11758    runtime: &SupervisorRuntimeConfig,
11759) -> Result<SupervisedChild, SuperviseError> {
11760    #[cfg(all(test, unix))]
11761    if let Some(source) = containment_tests::join_failure(spec, &runtime.child_roster) {
11762        containment::terminate_blocking(&runtime.child_roster.containment);
11763        return Err(SuperviseError::Spawn {
11764            program: spec.program.clone(),
11765            source,
11766            cgroup_path: None,
11767        });
11768    }
11769    let result = spawn_child_in_slot_async(spec, runtime, SpawnRole::Plain, false).await;
11770    if result.is_err() {
11771        if runtime.child_roster.admission_pending(&spec.module_id) {
11772            containment::terminate_deferred(&runtime.child_roster.containment);
11773        } else {
11774            containment::terminate_blocking(&runtime.child_roster.containment);
11775        }
11776    }
11777    result
11778}
11779
11780async fn spawn_child_in_slot_async(
11781    spec: &ModuleSpec,
11782    runtime: &SupervisorRuntimeConfig,
11783    role: SpawnRole,
11784    alternate_slot: bool,
11785) -> Result<SupervisedChild, SuperviseError> {
11786    let gate = runtime
11787        .config_delivery
11788        .as_ref()
11789        .map(|delivery| delivery.0.module_gate(&spec.module_id))
11790        .transpose()
11791        .map_err(|error| SuperviseError::InvalidSpec {
11792            reason: error.to_string(),
11793        })?;
11794    if gate.as_ref().is_some_and(|gate| gate.try_lock().is_err()) {
11795        return Err(SuperviseError::ConfigDeliveryDeferred {
11796            module_id: spec.module_id.clone(),
11797        });
11798    }
11799    let prepared_spec = spec.clone();
11800    let prepared_runtime = runtime.clone();
11801    let prepared = tokio::task::spawn_blocking(move || {
11802        let prepared = prepare_child_in_slot(
11803            &prepared_spec,
11804            prepared_runtime.connection_file_path.as_deref(),
11805            prepared_runtime.supervisor_handle.as_ref(),
11806            prepared_runtime.capture_logs_dir.as_deref(),
11807            &prepared_runtime.child_roster,
11808            prepared_runtime.spawn_delivery(),
11809            #[cfg(target_os = "linux")]
11810            prepared_runtime.cgroup_placement.as_ref(),
11811            role,
11812            alternate_slot,
11813            true,
11814        );
11815        prepared
11816    })
11817    .await
11818    .map_err(|_| SuperviseError::InvalidSpec {
11819        reason: "configuration launch preparation stopped".into(),
11820    })??;
11821    if let Some(registration) = &prepared.delivery_registration {
11822        if !registration.launch_current()? {
11823            return Err(spawn_plan_stale(spec));
11824        }
11825    }
11826    let _gate = match gate {
11827        Some(gate) => {
11828            Some(
11829                gate.try_lock_owned()
11830                    .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11831                        module_id: spec.module_id.clone(),
11832                    })?,
11833            )
11834        }
11835        None => None,
11836    };
11837    if let (Some(delivery), Some(admission)) = (runtime.spawn_delivery(), &prepared.spawn_admission)
11838    {
11839        delivery.check(spec, admission)?;
11840    }
11841    if prepared
11842        .spawn_admission
11843        .as_ref()
11844        .is_none_or(|admission| admission.launch.is_none())
11845        && prepared
11846            .program_binding
11847            .as_ref()
11848            .is_some_and(|binding| !binding.current())
11849    {
11850        return Err(spawn_plan_stale(spec));
11851    }
11852    // No await may separate this last identity check from ownership publication.
11853    let checked_base = prepared
11854        .spawn_admission
11855        .as_ref()
11856        .and_then(|admission| admission.approved_base())
11857        .cloned();
11858    let mut result = launch_child_in_slot(
11859        spec,
11860        &runtime.stderr_ring,
11861        &runtime.child_roster,
11862        prepared,
11863        runtime.register_timeout,
11864    );
11865    drop(_gate);
11866    if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11867        (runtime.config_delivery.as_ref(), &result)
11868    {
11869        if let Some(base) = checked_base.filter(|_| code == "program_refused") {
11870            let module = spec.module_id.clone();
11871            let identity = subc_config::raw_digest(detail.as_bytes());
11872            let recorded = delivery
11873                .0
11874                .on_blocking(move |owner| owner.record_program_refusal(&module, &base, identity))
11875                .await;
11876            match recorded {
11877                Ok(true) => {}
11878                Ok(false) => result = Err(spawn_plan_stale(spec)),
11879                Err(error) => {
11880                    result = Err(config_delivery::gate_hold_error(
11881                        spec,
11882                        error,
11883                        Some(delivery),
11884                    ))
11885                }
11886            }
11887        }
11888    }
11889    result
11890}
11891
11892fn validate_spawn_program(
11893    spec: &ModuleSpec,
11894    admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11895    delivery: Option<config_delivery::SpawnDelivery<'_>>,
11896) -> Result<(), SuperviseError> {
11897    if let Some(admission) = admission {
11898        if let Some(program) = &admission.launch {
11899            if let Err(error) = program.validate_for_spawn() {
11900                // A captured unsigned image is a definite refusal. Uncertain
11901                // reads are deferred by validate_for_spawn, not attributed to
11902                // the operator as a changed or unapprovable program.
11903                if let (Some(context), Some(base)) = (delivery, admission.approved_base()) {
11904                    if error.code == "program_refused"
11905                        && !context.record_refusal(spec, base, &error.detail)?
11906                    {
11907                        return Err(spawn_plan_stale(spec));
11908                    }
11909                }
11910                return Err(config_delivery::gate_hold_error(
11911                    spec,
11912                    error,
11913                    delivery.map(|context| context.runtime()).as_ref(),
11914                ));
11915            }
11916        }
11917    }
11918    Ok(())
11919}
11920
11921#[allow(clippy::too_many_arguments)]
11922fn prepare_child_in_slot(
11923    spec: &ModuleSpec,
11924    connection_file_path: Option<&std::path::Path>,
11925    handle: Option<&SupervisorHandle>,
11926    capture_logs_dir: Option<&std::path::Path>,
11927    roster: &ChildRoster,
11928    config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11929    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11930    role: SpawnRole,
11931    alternate_slot: bool,
11932    asynchronous: bool,
11933) -> Result<PreparedChild, SuperviseError> {
11934    require_program_search_path(spec)?;
11935    let spawn_admission = config_delivery
11936        .map(|delivery| delivery.prepare(spec))
11937        .transpose()?;
11938    #[cfg(target_os = "macos")]
11939    let gated = spawn_admission
11940        .as_ref()
11941        .is_some_and(|admission| admission.approved_base().is_some());
11942    let mut launch_spec = spawn_admission
11943        .as_ref()
11944        .and_then(|admission| admission.launch.as_ref())
11945        .map(|launch| launch.spec.clone())
11946        .unwrap_or_else(|| spec.clone());
11947    if launch_spec.script.is_some() {
11948        launch_spec.program = script::runner_path().map_err(|source| SuperviseError::Spawn {
11949            program: launch_spec.program.clone(),
11950            source,
11951            cgroup_path: None,
11952        })?;
11953    }
11954    let script_content = if let Some(path) = &launch_spec.script {
11955        Some(
11956            spawn_admission
11957                .as_ref()
11958                .and_then(|admission| admission.launch.as_ref())
11959                .and_then(|launch| launch.script.clone())
11960                .map_or_else(|| script::ScriptBytes::read(path), Ok)
11961                .map_err(|error| config_delivery::gate_hold_error(&launch_spec, error, None))?,
11962        )
11963    } else {
11964        None
11965    };
11966    let spec = &launch_spec;
11967    if let Some(error) = recovery_block_error(spec, roster) {
11968        return Err(error);
11969    }
11970    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
11971        return Err(previous_instance_error(spec, pid, reason));
11972    }
11973    if roster.is_closed() {
11974        return Err(SuperviseError::Spawn {
11975            program: spec.program.clone(),
11976            source: io::Error::other("the daemon is shutting down; not starting a new process"),
11977            cgroup_path: None,
11978        });
11979    }
11980    let carried = containment::current(&roster.containment);
11981    let carrying = carries_containment(spec.carry_containment, carried.is_some());
11982    if role != SpawnRole::Plain && carrying {
11983        return Err(SuperviseError::InvalidSpec {
11984            reason: "a carried container cannot overlap a swap candidate".to_owned(),
11985        });
11986    }
11987    #[cfg(unix)]
11988    let launch_identity = if carrying && carried.is_none() {
11989        Some(
11990            containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
11991                program: spec.program.clone(),
11992                source,
11993                cgroup_path: None,
11994            })?,
11995        )
11996    } else {
11997        None
11998    };
11999    #[cfg(target_os = "linux")]
12000    let cgroup_placement = match &carried {
12001        Some(container) => container.cgroup().map(|(placement, _)| placement),
12002        None => cgroup_placement,
12003    };
12004    #[cfg(unix)]
12005    let group_target = if let Some(container) = carried
12006        .as_ref()
12007        .filter(|container| container.group().is_some())
12008    {
12009        containment::GroupTarget::Pinned(Arc::clone(container))
12010    } else {
12011        #[cfg(target_os = "linux")]
12012        let cgroup_container = carrying && cgroup_placement.is_some();
12013        #[cfg(not(target_os = "linux"))]
12014        let cgroup_container = false;
12015        if cgroup_container {
12016            containment::GroupTarget::NoGroup
12017        } else {
12018            containment::GroupTarget::OwnLeader
12019        }
12020    };
12021    #[cfg(target_os = "linux")]
12022    let cgroup_name = {
12023        // Slot names alone are not kill domains: a retired incumbent may still
12024        // be draining when a later enable/restart spawns into the same slot.
12025        // Decimal entropy keeps the suffix unambiguous; Placement performs
12026        // the module-id escaping and constructs the filesystem path.
12027        if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
12028            name.to_owned()
12029        } else if cgroup_placement.is_none() {
12030            swap::cgroup_name(&spec.module_id, alternate_slot)
12031        } else {
12032            let nonce = generate_launch_nonce()?;
12033            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
12034            // Leave room for byte escaping and the suffix under NAME_MAX. The
12035            // label is only for humans; the nonce identifies the kill domain.
12036            let mut end = spec.module_id.len().min(64);
12037            while !spec.module_id.is_char_boundary(end) {
12038                end -= 1;
12039            }
12040            format!(
12041                "{}_{suffix}",
12042                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
12043            )
12044        }
12045    };
12046    #[cfg(not(target_os = "linux"))]
12047    let _ = alternate_slot;
12048    #[cfg(windows)]
12049    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
12050        program: spec.program.clone(),
12051        source,
12052        cgroup_path: None,
12053    })?;
12054    #[cfg(windows)]
12055    let program_binding = if asynchronous {
12056        let mut resolved = spec.clone();
12057        resolved.program = image_capture.path().to_path_buf();
12058        Some(SpawnProgramBinding::capture(&resolved)?)
12059    } else {
12060        None
12061    };
12062    #[cfg(not(windows))]
12063    let program_binding = (asynchronous || cfg!(target_os = "macos"))
12064        .then(|| SpawnProgramBinding::capture(spec))
12065        .transpose()?;
12066    #[cfg(target_os = "macos")]
12067    validate_spawn_program(spec, spawn_admission.as_ref(), config_delivery)?;
12068    #[cfg(target_os = "macos")]
12069    let (mut command, privacy_exec, exec_ack) = privacy_command(
12070        spec,
12071        program_binding
12072            .as_ref()
12073            .expect("macOS captures every launch"),
12074        roster,
12075        // Ungated launches retain legacy unsuspended exec, including scripts. Suspended
12076        // verification admits approved programs only. prepare() above fails
12077        // closed if the gate cannot decide; an image sample is not authority.
12078        gated,
12079    )?;
12080    #[cfg(windows)]
12081    let mut command = Command::new(image_capture.path());
12082    #[cfg(not(any(target_os = "macos", windows)))]
12083    let mut command = Command::new(
12084        program_binding
12085            .as_ref()
12086            .map_or(spec.program.as_path(), |binding| binding.path.as_path()),
12087    );
12088    command.args(&spec.args);
12089    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
12090    // that is the whole of the intent, so remove that one key rather than the
12091    // environment.
12092    //
12093    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
12094    // and took the POSIX environment with it. Modules spawned that way had no
12095    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
12096    // logging:
12097    //
12098    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
12099    //     both unset it fell back to the temp dir alone and `ck` could not find
12100    //     a daemon running on the same machine from inside any module's process
12101    //     tree — reporting a path the file has never lived at, which reads as
12102    //     "the daemon did not write its file".
12103    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
12104    //     the RELATIVE `.local/share`, so a module deriving its own store path
12105    //     resolved it against its own CWD. That is the store-fragmentation
12106    //     defect the daemon already refuses in config (`parse_doc` rejects a
12107    //     relative `storage.data_home`) arriving by derivation instead.
12108    //   * anything a module spawns inherited it: git without ~/.gitconfig,
12109    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
12110    //     quietly rather than erroring.
12111    //
12112    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
12113    // offered one candidate under /tmp while the file sat in /run/user/1000.
12114    //
12115    // A configured module is unaffected either way: `module_spec()` puts the
12116    // resolved CK_LOG into `spec.env`, which is applied below and therefore
12117    // wins over anything ambient.
12118    apply_child_env(&mut command, spec);
12119    if spec.script.is_some() {
12120        script::remove_runtime_env(&mut command, spec);
12121    }
12122    let origin_witness = crate::program_origin::LaunchWitness::capture_command(spec, &command)
12123        .map_err(|source| SuperviseError::Spawn {
12124            program: spec.program.clone(),
12125            source,
12126            cgroup_path: None,
12127        })?;
12128    command.current_dir(&origin_witness.working_directory);
12129    apply_spawn_role(&mut command, role);
12130    let nonce_handoff = apply_wire_spawn_args_for_admission(
12131        &mut command,
12132        spec,
12133        connection_file_path,
12134        handle,
12135        role,
12136        spawn_admission.as_ref(),
12137    )?;
12138    #[cfg(unix)]
12139    let mut slots = subc_os::registration_channel::ChildDescriptorSlots::default();
12140    command.env_remove(subc_os::script_pipe::SCRIPT_FD_ENV);
12141    let script_delivery = script_content
12142        .map(|content| {
12143            script::Delivery::prepare(
12144                content,
12145                &mut command,
12146                #[cfg(unix)]
12147                &mut slots,
12148            )
12149        })
12150        .transpose()
12151        .map_err(|source| SuperviseError::Spawn {
12152            program: spec.program.clone(),
12153            source,
12154            cgroup_path: None,
12155        })?;
12156    #[cfg(unix)]
12157    let registration_channel = if spec.protocol == ModuleProtocol::Subc {
12158        let (parent, child, metadata) = subc_os::registration_channel::registration_socketpair()
12159            .map_err(|source| SuperviseError::Spawn {
12160                program: spec.program.clone(),
12161                source,
12162                cgroup_path: None,
12163            })?;
12164        slots
12165            .channel(child)
12166            .map_err(|source| SuperviseError::Spawn {
12167                program: spec.program.clone(),
12168                source,
12169                cgroup_path: None,
12170            })?;
12171        command
12172            .env(subc_os::registration_channel::REGISTRATION_FD_ENV, metadata)
12173            .env(
12174                subc_os::registration_channel::DAEMON_PID_ENV,
12175                std::process::id().to_string(),
12176            );
12177        Some(parent)
12178    } else {
12179        None
12180    };
12181    #[cfg(windows)]
12182    let registration_channel = prepare_windows_registration_pipe(&mut command, spec)?;
12183    let binding_publication = if spawn_admission
12184        .as_ref()
12185        .is_some_and(|admission| admission.launch.is_some())
12186    {
12187        handle.and_then(|handle| handle.begin_program_binding(&spec.module_id, role))
12188    } else {
12189        None
12190    };
12191    let registration_spawn = (spec.protocol == ModuleProtocol::Subc)
12192        .then(|| {
12193            handle.map(|handle| {
12194                let spawn = RegistrationSpawn::new_with_binding_requirement(
12195                    handle,
12196                    &spec.module_id,
12197                    role,
12198                    binding_publication.clone(),
12199                    spawn_admission
12200                        .as_ref()
12201                        .is_some_and(|admission| admission.launch.is_some()),
12202                );
12203                if let Some(admission) = &spawn_admission {
12204                    admission.initialize_origin(&spawn.authority, &origin_witness);
12205                }
12206                handle.record_nonce_launch(&spawn);
12207                spawn
12208            })
12209        })
12210        .flatten();
12211
12212    #[cfg(target_os = "linux")]
12213    let cgroup_path = cgroup_placement
12214        .map(|placement| {
12215            if let Some(container) = &carried {
12216                let path = container
12217                    .cgroup_path()
12218                    .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
12219                placement.existing_module_path(path)?.ok_or_else(|| {
12220                    io::Error::new(
12221                        io::ErrorKind::NotFound,
12222                        "carried cgroup disappeared before replacement",
12223                    )
12224                })
12225            } else {
12226                if carrying {
12227                    containment::prepare_cgroup_path(placement, &cgroup_name)
12228                } else {
12229                    placement.module_path(&cgroup_name)
12230                }
12231            }
12232        })
12233        .transpose()
12234        .map_err(|source| SuperviseError::Cgroup {
12235            module_id: spec.module_id.clone(),
12236            source,
12237        })?;
12238    #[cfg(not(target_os = "linux"))]
12239    let cgroup_path: Option<PathBuf> = None;
12240    // Own a newly created directory before any fallible admission or command
12241    // preparation. Existing carried containment must never be removed here.
12242    #[cfg(target_os = "linux")]
12243    let cgroup_cleanup = PreparedCgroup(
12244        cgroup_placement
12245            .filter(|_| carried.is_none())
12246            .map(|placement| (placement.clone(), cgroup_name.clone())),
12247    );
12248    #[cfg(unix)]
12249    let mut admission_gate = if carrying {
12250        Some(
12251            admission::Gate::install(&mut command, spec, roster).map_err(|source| {
12252                SuperviseError::Spawn {
12253                    program: spec.program.clone(),
12254                    source,
12255                    cgroup_path: cgroup_path.clone(),
12256                }
12257            })?,
12258        )
12259    } else {
12260        None
12261    };
12262    #[cfg(target_os = "linux")]
12263    if let Some(path) = &cgroup_path {
12264        apply_cgroup_placement(&mut command, spec, path)?;
12265    }
12266
12267    let output_sink = if let Some(logs_dir) = capture_logs_dir {
12268        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
12269        match ChildOutputSink::open(&path, capture_retention(spec)) {
12270            Ok(sink) => sink,
12271            Err(error) => {
12272                warn!(
12273                    module_id = %spec.module_id,
12274                    path = %path.display(),
12275                    error = %error,
12276                    "could not open child output capture file; forwarding to stderr"
12277                );
12278                ChildOutputSink::Stderr
12279            }
12280        }
12281    } else {
12282        ChildOutputSink::Stderr
12283    };
12284
12285    command.stdout(Stdio::piped());
12286    command.stderr(Stdio::piped());
12287    command.kill_on_drop(!carrying);
12288    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
12289    // before exec). In the daemon's group, a service manager that kills the
12290    // job's process group when the daemon exits (launchd's default) killed
12291    // every module at the same moment its control connection closed, so no
12292    // module ever ran its EOF teardown on a daemon stop. Outside that group a
12293    // module is reached only by the daemon: the EOF it sees when its
12294    // connection closes, and the bounded stop in `child_roster` for anything
12295    // still running after that. On Linux this composes with the cgroup
12296    // placement above: that is a pre_exec write to cgroup.procs, std performs
12297    // setpgid in the child before running pre_exec callbacks, and the two
12298    // change independent process attributes.
12299    //
12300    // stdin is /dev/null because a process outside the terminal's foreground
12301    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
12302    // by hand would otherwise hand down. Under a service manager stdin is
12303    // already /dev/null.
12304    #[cfg(unix)]
12305    command.process_group(
12306        carried
12307            .as_ref()
12308            .map(|container| container.verified_join_group())
12309            .transpose()
12310            .map_err(|source| SuperviseError::Spawn {
12311                program: spec.program.clone(),
12312                source,
12313                cgroup_path: cgroup_path.clone(),
12314            })?
12315            .flatten()
12316            .unwrap_or(0) as i32,
12317    );
12318    command.stdin(Stdio::null());
12319    #[cfg(unix)]
12320    if let Some(gate) = &mut admission_gate {
12321        gate.install_placement(&mut command);
12322    }
12323    // Fill all reserved slots in one allocation-free step, after placement has
12324    // stopped using descriptors that the fixed slots may replace.
12325    #[cfg(unix)]
12326    {
12327        if let Some(handoff) = nonce_handoff {
12328            slots.nonce(handoff);
12329        }
12330        #[cfg(target_os = "macos")]
12331        slots
12332            .acknowledgement(exec_ack.into_descriptor())
12333            .map_err(|source| SuperviseError::Spawn {
12334                program: spec.program.clone(),
12335                source,
12336                cgroup_path: cgroup_path.clone(),
12337            })?;
12338        slots.install(command.as_std_mut());
12339    }
12340    #[cfg(not(any(unix, windows)))]
12341    let _ = nonce_handoff;
12342
12343    // Containment, step 1 of 3 (issue #109): create the child suspended so it
12344    // cannot run a single instruction -- and therefore cannot spawn a
12345    // grandchild -- before it is in the job. See `contain_spawned_child` for the
12346    // other two steps and why the window matters.
12347    #[cfg(windows)]
12348    {
12349        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
12350            program: spec.program.clone(),
12351            source,
12352            cgroup_path: None,
12353        })?;
12354        subc_jobobject::suspend_on_create_async(&mut command);
12355    }
12356    // Timestamp before the spawn call: on Unix the child can serve before that
12357    // call even returns. This conservative bound never labels an older child
12358    // as a replacement for a request received during process creation.
12359    let spawned_at = Instant::now();
12360    let delivery_context = config_delivery;
12361    let config_delivery = config_delivery
12362        .map(|delivery| {
12363            delivery.spawned(
12364                spec,
12365                spawn_admission
12366                    .as_ref()
12367                    .expect("delivery prepares spawn admission"),
12368                spawned_at,
12369                role,
12370                registration_spawn
12371                    .as_ref()
12372                    .map(|spawn| spawn.authority.clone()),
12373            )
12374        })
12375        .transpose()?
12376        .flatten();
12377    let mut delivery_registration =
12378        delivery_context.map(|context| context.registration_guard(config_delivery.as_ref(), role));
12379    if let Some(registration) = &mut delivery_registration {
12380        registration.defer_cleanup();
12381    }
12382    if asynchronous {
12383        #[cfg(test)]
12384        if let Some(context) = delivery_context {
12385            context.pause_preflight_for_test();
12386        }
12387    }
12388    validate_spawn_program(spec, spawn_admission.as_ref(), delivery_context)?;
12389    Ok(PreparedChild {
12390        script_delivery,
12391        registration_spawn,
12392        registration_handle: handle.cloned(),
12393        #[cfg(any(unix, windows))]
12394        registration_channel,
12395        config_context: delivery_context.map(|delivery| delivery.runtime()),
12396        binding_publication,
12397        launch_spec,
12398        spawn_admission,
12399        command,
12400        carrying,
12401        carried: carried.clone(),
12402        #[cfg(unix)]
12403        launch_identity,
12404        #[cfg(unix)]
12405        group_target,
12406        #[cfg(target_os = "linux")]
12407        cgroup_placement: cgroup_placement.cloned(),
12408        #[cfg(target_os = "linux")]
12409        cgroup_cleanup,
12410        #[cfg(target_os = "linux")]
12411        cgroup_name,
12412        cgroup_path,
12413        #[cfg(unix)]
12414        admission_gate,
12415        output_sink,
12416        #[cfg(target_os = "macos")]
12417        privacy_exec,
12418        #[cfg(windows)]
12419        image_capture,
12420        #[cfg(windows)]
12421        nonce_handoff,
12422        spawned_at,
12423        config_delivery,
12424        delivery_registration,
12425        program_binding,
12426    })
12427}
12428
12429fn launch_child_in_slot(
12430    _spec: &ModuleSpec,
12431    ring: &Arc<Mutex<StderrRing>>,
12432    roster: &ChildRoster,
12433    prepared: PreparedChild,
12434    register_timeout: Duration,
12435) -> Result<SupervisedChild, SuperviseError> {
12436    let launch_spec = prepared.launch_spec.clone();
12437    let registration_declaration = prepared
12438        .spawn_admission
12439        .as_ref()
12440        .and_then(|admission| admission.registration_declaration(&launch_spec.module_id));
12441    let admission_base = prepared
12442        .spawn_admission
12443        .as_ref()
12444        .and_then(|admission| admission.approved_base())
12445        .cloned();
12446    let spec = &launch_spec;
12447    let expected_image = prepared
12448        .spawn_admission
12449        .as_ref()
12450        .and_then(|admission| admission.launch.as_ref())
12451        .map(|launch| launch.image);
12452    let program_launch = prepared
12453        .spawn_admission
12454        .and_then(|admission| admission.launch);
12455    let config_owner = prepared
12456        .config_context
12457        .as_ref()
12458        .map(|context| context.0.clone());
12459    let binding_publication = prepared.binding_publication;
12460    #[cfg(test)]
12461    if let Some(context) = &prepared.config_context {
12462        context.0.before_spawn_exec_for_test();
12463    }
12464    #[cfg(feature = "test-support")]
12465    if let Some(context) = &prepared.config_context {
12466        context
12467            .0
12468            .pause_spawn_before_exec()
12469            .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(context)))?;
12470    }
12471    let mut command = prepared.command;
12472    let carrying = prepared.carrying;
12473    let carried = prepared.carried;
12474    #[cfg(unix)]
12475    let launch_identity = prepared.launch_identity;
12476    #[cfg(unix)]
12477    let group_target = prepared.group_target;
12478    #[cfg(target_os = "linux")]
12479    let cgroup_placement = prepared.cgroup_placement.as_ref();
12480    #[cfg(target_os = "linux")]
12481    let cgroup_name = prepared.cgroup_name;
12482    #[cfg(target_os = "linux")]
12483    let mut cgroup_cleanup = prepared.cgroup_cleanup;
12484    #[cfg(target_os = "linux")]
12485    let _ = cgroup_cleanup.0.take();
12486    let cgroup_path = prepared.cgroup_path;
12487    #[cfg(unix)]
12488    let admission_gate = prepared.admission_gate;
12489    let output_sink = prepared.output_sink;
12490    #[cfg(target_os = "macos")]
12491    let mut privacy_exec = prepared.privacy_exec;
12492    #[cfg(target_os = "macos")]
12493    if let Some(pending) = &mut privacy_exec {
12494        pending.admission_binding = binding_publication.clone();
12495        if expected_image.is_some() {
12496            pending.expected = expected_image;
12497        }
12498    }
12499    #[cfg(any(unix, windows))]
12500    let registration_channel = prepared.registration_channel;
12501    let registration_spawn = prepared.registration_spawn;
12502    let registration_handle = prepared.registration_handle;
12503    #[cfg(windows)]
12504    let image_capture = prepared.image_capture;
12505    #[cfg(windows)]
12506    let nonce_handoff = prepared.nonce_handoff;
12507    let spawned_at = prepared.spawned_at;
12508    let config_delivery = prepared.config_delivery;
12509    let delivery_registration = prepared.delivery_registration;
12510    let _program_binding = prepared.program_binding;
12511    #[cfg(all(test, unix))]
12512    containment_tests::record_spawn_attempt(spec, &mut command);
12513    #[cfg(unix)]
12514    let spawned = match admission_gate {
12515        Some(gate) => gate
12516            .spawn(
12517                command,
12518                spec,
12519                roster,
12520                launch_identity,
12521                #[cfg(target_os = "linux")]
12522                cgroup_placement
12523                    .zip(cgroup_path.as_ref())
12524                    .map(|(placement, path)| {
12525                        (placement.clone(), cgroup_name.clone(), path.clone())
12526                    }),
12527            )
12528            .map(|(child, admitted)| (child, Some(admitted))),
12529        None => {
12530            let spawned =
12531                subc_os::child_spawn::tokio_spawn(&mut command).map(|child| (child, None));
12532            // The callback owns all child-only sources. Closing them here is
12533            // necessary for channel EOF and macOS exec acknowledgement EOF.
12534            drop(command);
12535            spawned
12536        }
12537    };
12538    #[cfg(not(unix))]
12539    let spawned = subc_os::child_spawn::tokio_spawn(&mut command);
12540    let spawned = match spawned {
12541        Ok(child) => child,
12542        Err(source) => {
12543            if carried
12544                .as_ref()
12545                .and_then(|container| container.group())
12546                .is_some()
12547            {
12548                containment::terminate_blocking(&roster.containment);
12549            }
12550            #[cfg(target_os = "linux")]
12551            if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12552                remove_module_cgroup(placement, &cgroup_name);
12553            }
12554            return Err(SuperviseError::Spawn {
12555                program: spec.program.clone(),
12556                source,
12557                cgroup_path,
12558            });
12559        }
12560    };
12561    #[cfg(unix)]
12562    let (mut child, mut admitted) = spawned;
12563    #[cfg(unix)]
12564    let group_target = admitted
12565        .as_ref()
12566        .map(|attempt| attempt.group_target())
12567        .unwrap_or(group_target);
12568    #[cfg(not(unix))]
12569    let mut child = spawned;
12570    if let Some(delivery) = prepared.script_delivery {
12571        delivery.start(child.id().expect("spawned child has a pid"));
12572    }
12573    #[cfg(test)]
12574    spawn_record_tests::after_create(&child, spec);
12575
12576    // Bind while the child is still suspended, before job assignment resumes it.
12577    // Keeping the executable handle open across CreateProcessW denies rename
12578    // and deletion, so the file cannot be exchanged during spawn. Bind that
12579    // file ID to the process handle returned by spawn, never a reopened PID.
12580    #[cfg(windows)]
12581    let windows_image = match image_capture.bind_tokio(&child) {
12582        Ok(image) => Arc::new(image),
12583        Err(source) => {
12584            kill_and_reap_rejected_child(&mut child);
12585            return Err(SuperviseError::Spawn {
12586                program: spec.program.clone(),
12587                source,
12588                cgroup_path: None,
12589            });
12590        }
12591    };
12592    // Assign the suspended child to its module job; resume only after its
12593    // process identity has been durably recorded for orphan recovery.
12594    #[cfg(windows)]
12595    let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
12596    let spawned_at_ms = unix_ms_now();
12597    #[cfg(not(windows))]
12598    let spawned_from = spec.program.clone();
12599    #[cfg(windows)]
12600    let spawned_from = windows_image.path().to_path_buf();
12601    #[cfg(not(windows))]
12602    let spawned_file_identity = spawned_file_identity(&spawned_from);
12603    #[cfg(windows)]
12604    let spawned_file_identity = Some(SpawnedFileIdentity {
12605        device: 0,
12606        inode: 0,
12607        windows_image: Some(Arc::clone(&windows_image)),
12608    });
12609    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
12610        program: spec.program.clone(),
12611        source: io::Error::other("spawned child exposed no live pid"),
12612        cgroup_path: cgroup_path.clone(),
12613    })?;
12614    #[cfg(windows)]
12615    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
12616    #[cfg(not(windows))]
12617    let process_start_time = admitted
12618        .as_ref()
12619        .map(|admitted| admitted.birth)
12620        .or_else(|| crate::provenance::process_start_time(pid));
12621    #[cfg(windows)]
12622    let process_start_time = Some(windows_image.start_time());
12623    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
12624    #[cfg(all(test, target_os = "macos"))]
12625    privacy_exec_boundary_tests::before_image_sample(spec, pid);
12626    #[cfg(all(test, target_os = "linux"))]
12627    assert!(roster.admission_event_for_test(&spec.module_id, "before_image_binding", Some(pid)));
12628    // Record the committed kernel image, not the configured pathname: PATH
12629    // lookup and shebang interpretation can select a different executable.
12630    // Linux's posix_spawn can return before that image is installed, so the
12631    // observer also waits for the kernel's first-exec flag to clear.
12632    #[cfg(not(windows))]
12633    let recorded_image = observe_spawned_image(pid);
12634    #[cfg(all(test, target_os = "linux"))]
12635    let recorded_image = if roster.admission_event_for_test(
12636        &spec.module_id,
12637        "after_spawn_image_sample",
12638        Some(pid),
12639    ) {
12640        recorded_image
12641    } else {
12642        None
12643    };
12644    #[cfg(target_os = "linux")]
12645    // An exited child cannot receive HELLO service and has no readable /proc/exe.
12646    // Keep its real exit for the supervisor instead of turning a crash into a hold.
12647    let exited_before_binding = expected_image.is_some()
12648        && recorded_image.and_then(|image| image.executable) != expected_image
12649        && child.try_wait().ok().flatten().is_some();
12650    #[cfg(target_os = "linux")]
12651    let unavailable_initial_image = expected_image.is_some()
12652        && recorded_image.and_then(|image| image.executable).is_none()
12653        && !exited_before_binding;
12654    #[cfg(not(target_os = "linux"))]
12655    let unavailable_initial_image = false;
12656    #[cfg(target_os = "linux")]
12657    if expected_image
12658        .is_some_and(|expected| recorded_image.and_then(|image| image.executable) != Some(expected))
12659        && !exited_before_binding
12660        && !unavailable_initial_image
12661    {
12662        kill_module_cgroup(cgroup_placement, &cgroup_name);
12663        kill_and_reap_rejected_child(&mut child, group_target.clone());
12664        return Err(SuperviseError::ConfigHeld {
12665            module_id: spec.module_id.clone(),
12666            code: "program_refused".into(),
12667            detail: format!(
12668                "spawned executable {:?} does not match hashed program {:?}",
12669                recorded_image.and_then(|image| image.executable),
12670                expected_image
12671            ),
12672        });
12673    }
12674    #[cfg(not(target_os = "linux"))]
12675    let _ = expected_image;
12676    #[cfg(target_os = "macos")]
12677    let binding_ready = privacy_exec.is_none();
12678    #[cfg(target_os = "linux")]
12679    let binding_ready = !exited_before_binding;
12680    #[cfg(not(any(target_os = "macos", target_os = "linux")))]
12681    let binding_ready = cfg!(windows);
12682    // spawn() confirms only the first exec, into the trampoline. Never persist
12683    // the trampoline image; the asynchronous acknowledgement publishes the
12684    // module image once the trampoline has replaced itself with the module.
12685    #[cfg(target_os = "macos")]
12686    let recorded_image = if privacy_exec.is_some() {
12687        None
12688    } else {
12689        recorded_image
12690    };
12691    #[cfg(target_os = "linux")]
12692    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
12693    #[cfg(not(target_os = "linux"))]
12694    let recorded_cgroup_name = None;
12695    #[cfg(unix)]
12696    let preadmitted = admitted
12697        .as_mut()
12698        .and_then(|admitted| admitted.leader.take());
12699    #[cfg(not(unix))]
12700    let preadmitted: Option<crate::child_roster::RosterGuard> = None;
12701    let roster_guard = if let Some(guard) = preadmitted {
12702        Ok(guard)
12703    } else {
12704        roster.admit(
12705            spec.module_id.clone(),
12706            pid,
12707            spec.protocol,
12708            process_start_time,
12709            crate::child_roster::RecordedIdentity {
12710                container: None,
12711                #[cfg(unix)]
12712                carried: None,
12713                #[cfg(not(windows))]
12714                start_time: recorded_image.map(|image| image.start_time),
12715                #[cfg(windows)]
12716                start_time: process_start_time,
12717                #[cfg(not(windows))]
12718                executable: recorded_image
12719                    .and_then(|image| image.executable)
12720                    .map(crate::live_children::ExecutableIdentity::from),
12721                #[cfg(windows)]
12722                executable: Some(crate::live_children::ExecutableIdentity::from(
12723                    windows_image.identity(),
12724                )),
12725                cgroup_name: recorded_cgroup_name,
12726                #[cfg(target_os = "linux")]
12727                cgroup_placement: cgroup_placement.cloned(),
12728            },
12729        )
12730    };
12731    let roster_guard = match roster_guard {
12732        Ok(guard) => guard,
12733        Err(failure) => {
12734            return failure.with_cleanup(|source| {
12735                #[cfg(windows)]
12736                if let Some(job) = &job {
12737                    let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12738                }
12739                #[cfg(target_os = "linux")]
12740                kill_module_cgroup(cgroup_placement, &cgroup_name);
12741                #[cfg(unix)]
12742                let rejected_group = group_target.clone();
12743                kill_and_reap_rejected_child(
12744                    &mut child,
12745                    #[cfg(unix)]
12746                    rejected_group,
12747                );
12748                #[cfg(target_os = "linux")]
12749                if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12750                    remove_module_cgroup(placement, &cgroup_name);
12751                }
12752                Err(SuperviseError::Spawn {
12753                    program: spec.program.clone(),
12754                    source: io::Error::other(format!(
12755                        "could not durably record module child: {source}"
12756                    )),
12757                    cgroup_path,
12758                })
12759            });
12760        }
12761    };
12762    #[cfg(unix)]
12763    if admitted.is_some() {
12764        if let Some(image) = recorded_image {
12765            if let Err(source) = roster_guard.confirm_executable(image) {
12766                kill_and_reap_rejected_child(&mut child, group_target.clone());
12767                return Err(SuperviseError::Spawn {
12768                    program: spec.program.clone(),
12769                    source,
12770                    cgroup_path,
12771                });
12772            }
12773        }
12774    }
12775    // The pre-spawn check can pass just before shutdown begins. Ownership is
12776    // published at the start of admission, even while a record write is queued.
12777    // If shutdown already saw an empty roster before that publication, this
12778    // check sees the roster closed and ends the child before starting a module.
12779    if roster.is_closed() {
12780        if carrying {
12781            #[cfg(unix)]
12782            let rejected_group = admitted
12783                .as_ref()
12784                .map(|admitted| admitted.rejection_target())
12785                .unwrap_or_else(|| group_target.clone());
12786            #[cfg(windows)]
12787            if let Some(job) = &job {
12788                let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12789            }
12790            kill_and_reap_rejected_child(
12791                &mut child,
12792                #[cfg(unix)]
12793                rejected_group,
12794            );
12795            #[cfg(all(test, unix))]
12796            let _ =
12797                roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
12798            drop(roster_guard);
12799            return Err(SuperviseError::Spawn {
12800                program: spec.program.clone(),
12801                source: io::Error::other(
12802                    "the daemon began shutting down during carried admission; ended the container",
12803                ),
12804                cgroup_path,
12805            });
12806        }
12807        // This child was never admitted, so there is no module protocol shutdown to wait for.
12808        #[cfg(target_os = "linux")]
12809        kill_module_cgroup(cgroup_placement, &cgroup_name);
12810        if let Err(error) = child.start_kill() {
12811            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
12812        }
12813        #[cfg(target_os = "linux")]
12814        if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12815            // This spawn was never admitted, so shutdown has no roster entry
12816            // to await. Do not detach its cleanup: the runtime could exit
12817            // before that task reaps the rejected child and removes its group.
12818            while matches!(child.try_wait(), Ok(None)) {
12819                std::thread::yield_now();
12820            }
12821            if matches!(
12822                subc_cgroup::kill_module(Some(placement), &cgroup_name),
12823                subc_cgroup::KillOutcome::Killed
12824            ) {
12825                if let Ok(path) = placement.module_path(&cgroup_name) {
12826                    while std::fs::read_to_string(path.join("cgroup.events"))
12827                        .ok()
12828                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
12829                    {
12830                        std::thread::yield_now();
12831                    }
12832                }
12833            }
12834            remove_module_cgroup(placement, &cgroup_name);
12835        }
12836        drop(roster_guard);
12837        return Err(SuperviseError::Spawn {
12838            program: spec.program.clone(),
12839            source: io::Error::other(
12840                "the daemon began shutting down while this process was starting; ended it",
12841            ),
12842            cgroup_path,
12843        });
12844    }
12845
12846    let stdout_pump = match child.stdout.take() {
12847        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
12848        None => {
12849            warn!(
12850                module_id = %spec.module_id,
12851                "spawned child exposed no stdout pipe; file capture will be incomplete"
12852            );
12853            None
12854        }
12855    };
12856    let stderr_pump = match child.stderr.take() {
12857        Some(stderr) => {
12858            let generation = ring
12859                .lock()
12860                .unwrap_or_else(|poisoned| poisoned.into_inner())
12861                .begin_process();
12862            Some(StderrPump {
12863                task: tokio::spawn(pump_stderr_to(
12864                    stderr,
12865                    Arc::clone(ring),
12866                    generation,
12867                    output_sink,
12868                )),
12869                generation,
12870            })
12871        }
12872        None => {
12873            // Spawning succeeded but the pipe did not materialise. Recording it as
12874            // uncaptured keeps the tail honest: the alternative is an empty tail
12875            // that reads as a module which printed nothing.
12876            ring.lock()
12877                .unwrap_or_else(|poisoned| poisoned.into_inner())
12878                .mark_not_captured("stderr pipe was not available on spawn");
12879            warn!(
12880                module_id = %spec.module_id,
12881                "spawned child exposed no stderr pipe; tail will be unavailable"
12882            );
12883            None
12884        }
12885    };
12886
12887    let exec = Arc::new(crate::activity::ExecOutcome::new());
12888    #[cfg(target_os = "macos")]
12889    let pending_exec = privacy_exec.is_some();
12890    #[cfg(not(target_os = "macos"))]
12891    let pending_exec = false;
12892    if !pending_exec {
12893        exec.finish(crate::activity::ExecState::CommittedExec);
12894    }
12895    #[cfg(windows)]
12896    let subtree_target = job
12897        .as_ref()
12898        .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
12899        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
12900    #[cfg(target_os = "linux")]
12901    let subtree_target = cgroup_path
12902        .map(crate::subtree_resources::Target::Cgroup)
12903        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
12904    #[cfg(not(any(target_os = "linux", windows)))]
12905    let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
12906    let mut supervised = SupervisedChild {
12907        registration_spawn,
12908        #[cfg(any(unix, windows))]
12909        registration_channel,
12910        live_program_binding: None,
12911        registered_config_connection: None,
12912        admission_base,
12913        registration_declaration,
12914        _program_binding: binding_publication,
12915        subtree_target,
12916        child: containment::ProcessChild::plain(child, pid),
12917        exec: crate::activity::ExecOwner(exec),
12918        #[cfg(windows)]
12919        nonce_delivery,
12920        protocol: spec.protocol,
12921        #[cfg(target_os = "linux")]
12922        module_id: cgroup_name,
12923        #[cfg(target_os = "linux")]
12924        cgroup_placement: cgroup_placement.cloned(),
12925        #[cfg(windows)]
12926        job,
12927        stdout_pump,
12928        stderr_pump,
12929        stderr_ring: Arc::clone(ring),
12930        spawned_at,
12931        config_delivery,
12932        registration_deadline: None,
12933        registration_watch: None,
12934        spawned_at_ms,
12935        spawned_from,
12936        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
12937        spawned_file_identity,
12938        process_start_time,
12939        process_identity,
12940        pid,
12941        roster_guard: Some(roster_guard),
12942        #[cfg(target_os = "macos")]
12943        privacy_exec,
12944        #[cfg(target_os = "macos")]
12945        report_ready: Arc::new(OnceLock::new()),
12946        spawn_failure: None,
12947        admission_kill_sent: false,
12948    };
12949    if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
12950        supervised.child.reject(
12951            #[cfg(unix)]
12952            group_target,
12953        );
12954        #[cfg(all(test, unix))]
12955        let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
12956        return Err(error);
12957    }
12958    if let (Some(launch), Some(owner), Some(base)) = (
12959        program_launch,
12960        config_owner,
12961        supervised.admission_base.clone(),
12962    ) {
12963        supervised.live_program_binding =
12964            content_binding::LiveProgramBinding::new(launch, &supervised, owner, base);
12965        if let (Some(publication), Some(live)) = (
12966            &supervised._program_binding,
12967            &supervised.live_program_binding,
12968        ) {
12969            publication.attach(live);
12970        }
12971    }
12972    #[cfg(unix)]
12973    if let Some(admitted) = admitted.take() {
12974        admitted.disarm();
12975    }
12976    #[cfg(windows)]
12977    if let Err(error) = resume_suspended_child(pid, spec) {
12978        containment::terminate_blocking(&roster.containment);
12979        let _ = supervised.child.start_kill();
12980        return Err(error);
12981    }
12982    if binding_ready {
12983        // An unreadable initial image is not a definite mismatch. Keep the
12984        // owned child and delivery registration until its reaped status decides
12985        // whether it crashed on its own or was stopped by admission's kill.
12986        let checked = if unavailable_initial_image {
12987            Err(crate::config_gate::GateError::new(
12988                "config_delivery_deferred",
12989                "initial process image is unavailable",
12990            ))
12991        } else {
12992            match (
12993                &supervised._program_binding,
12994                &supervised.live_program_binding,
12995            ) {
12996                (Some(binding), _) => binding.verify(),
12997                (None, Some(live)) => live.check(),
12998                _ if supervised.admission_base.is_some() => {
12999                    Err(crate::config_gate::GateError::new(
13000                        "config_delivery_deferred",
13001                        "live program witness is unavailable",
13002                    ))
13003                }
13004                _ => Ok(()),
13005            }
13006        };
13007        #[cfg(all(test, target_os = "linux"))]
13008        let checked = if roster.admission_event_for_test(
13009            &spec.module_id,
13010            "after_content_witness_check",
13011            Some(pid),
13012        ) {
13013            checked
13014        } else {
13015            Err(crate::config_gate::GateError::new(
13016                "config_delivery_deferred",
13017                "current process image is unavailable",
13018            ))
13019        };
13020        if let Err(error) = checked {
13021            if let Some(binding) = &mut supervised._program_binding {
13022                binding.reject();
13023            }
13024            // Reap through the normal child-reaping path, retaining the real exit record and
13025            // its roster ownership. A refusal must also end any helpers.
13026            if error.code == "program_refused" || supervised.try_wait().ok().flatten().is_none() {
13027                supervised.spawn_failure = Some(format!("{}: {}", error.code, error.detail));
13028                #[cfg(all(test, target_os = "linux"))]
13029                assert!(roster.admission_event_for_test(
13030                    &spec.module_id,
13031                    if unavailable_initial_image {
13032                        "before_initial_image_kill"
13033                    } else {
13034                        "before_content_witness_kill"
13035                    },
13036                    Some(pid),
13037                ));
13038                let _ = supervised.start_admission_kill();
13039                #[cfg(all(test, target_os = "linux"))]
13040                if supervised.admission_kill_sent {
13041                    assert!(roster.admission_event_for_test(
13042                        &spec.module_id,
13043                        "after_admission_kill_sent",
13044                        Some(pid),
13045                    ));
13046                }
13047            }
13048        }
13049    }
13050    if let Some(registration) = delivery_registration {
13051        registration.commit();
13052    }
13053    if let Some(spawn) = &supervised.registration_spawn {
13054        if let Some(identity) = supervised.process_identity {
13055            let _ = spawn.identity.set(identity);
13056        }
13057        if supervised.spawn_failure.is_some() {
13058            spawn.refuse();
13059        } else if binding_ready {
13060            spawn.ready();
13061        }
13062        #[cfg(any(unix, windows))]
13063        if let (Some(handle), Some(channel)) =
13064            (registration_handle, supervised.registration_channel.take())
13065        {
13066            handle.serve_registration(channel, spawn.clone(), pid, spawned_at + register_timeout);
13067        }
13068    }
13069    Ok(supervised)
13070}
13071
13072#[cfg(test)]
13073mod spawn_record_tests {
13074    use super::*;
13075
13076    thread_local! {
13077        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
13078    }
13079
13080    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
13081        LAST_PID.set(child.id());
13082        #[cfg(windows)]
13083        if let Some((_, address)) = _spec
13084            .env
13085            .iter()
13086            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
13087        {
13088            use std::io::{Read, Write};
13089            let mut stream = std::net::TcpStream::connect(address).unwrap();
13090            stream
13091                .write_all(&child.id().unwrap().to_ne_bytes())
13092                .unwrap();
13093            let mut release = [0];
13094            stream.read_exact(&mut release).unwrap();
13095        }
13096    }
13097
13098    #[cfg(windows)]
13099    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
13100        if spec
13101            .env
13102            .iter()
13103            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
13104        {
13105            Err(io::Error::other("injected module job assignment failure"))
13106        } else {
13107            Ok(())
13108        }
13109    }
13110
13111    fn spec() -> ModuleSpec {
13112        ModuleSpec {
13113            script: None,
13114            module_id: "spawn-record-worker".into(),
13115            #[cfg(unix)]
13116            program: "/bin/sleep".into(),
13117            #[cfg(windows)]
13118            program: std::env::current_exe().unwrap(),
13119            #[cfg(unix)]
13120            args: vec!["60".into()],
13121            #[cfg(windows)]
13122            args: vec![
13123                "--exact".into(),
13124                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
13125                "--nocapture".into(),
13126            ],
13127            env: vec![],
13128            reserved: false,
13129            reserved_prefixes: vec![],
13130            protocol: ModuleProtocol::None,
13131            overlap: Default::default(),
13132            carry_containment: false,
13133        }
13134    }
13135
13136    #[tokio::test]
13137    async fn record_sync_failure_fails_spawn_and_reaps_child() {
13138        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
13139        let supervisor =
13140            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
13141                .with_live_children_record(home.join("live-children.json"));
13142        crate::live_children::fail_next_file_sync();
13143        let result = supervisor.spawn(spec());
13144        let failed = result.is_err();
13145        if let Ok(module) = result {
13146            module.stop().await.unwrap();
13147        }
13148        assert!(failed, "record failure must fail spawn");
13149        let pid = LAST_PID.get().expect("the real OS child was created");
13150        assert!(
13151            subc_os::Process::open(pid).unwrap().is_none(),
13152            "child {pid} survived rejected admission or was not reaped"
13153        );
13154    }
13155
13156    #[cfg(target_os = "macos")]
13157    #[test]
13158    #[ignore = "manual on-device latency measurement"]
13159    fn durable_record_write_latency_50() {
13160        let home = cortexkit_test_support::ScratchDir::new("record-latency");
13161        let path = home.join("live-children.json");
13162        let children = vec![crate::live_children::LiveChild {
13163            module_id: "latency-worker".into(),
13164            pid: std::process::id(),
13165            protocol: ModuleProtocol::None,
13166            start_time: Some(1),
13167            executable: None,
13168            cgroup_name: None,
13169            container: None,
13170        }];
13171        let mut samples = Vec::new();
13172        for _ in 0..50 {
13173            let start = std::time::Instant::now();
13174            crate::live_children::write_record(&path, &children).unwrap();
13175            samples.push(start.elapsed());
13176        }
13177        samples.sort();
13178        eprintln!(
13179            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
13180            (samples[24] + samples[25]) / 2,
13181            samples[47]
13182        );
13183    }
13184
13185    #[cfg(windows)]
13186    #[test]
13187    fn windows_store_touch_fixture() {
13188        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
13189            std::fs::write(path, b"module ran").unwrap();
13190        }
13191    }
13192
13193    #[cfg(windows)]
13194    #[tokio::test]
13195    async fn windows_daemon_fixture() {
13196        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
13197            return;
13198        };
13199        let mut spec = spec();
13200        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
13201        let supervisor =
13202            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13203        let _module = supervisor.spawn(spec).unwrap();
13204    }
13205
13206    #[cfg(windows)]
13207    #[test]
13208    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
13209        use std::io::Read;
13210        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13211        // An outer job models a scheduler/service host. The daemon's root and
13212        // root job must nest without requiring breakaway from this job.
13213        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13214        command
13215            .args([
13216                "--exact",
13217                "supervise::spawn_record_tests::windows_daemon_fixture",
13218                "--nocapture",
13219            ])
13220            .env(
13221                "SUBC_TEST_DAEMON_BARRIER",
13222                listener.local_addr().unwrap().to_string(),
13223            );
13224        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13225        listener.set_nonblocking(true).unwrap();
13226        let deadline = std::time::Instant::now() + Duration::from_secs(30);
13227        let mut stream = loop {
13228            if let Ok((stream, _)) = listener.accept() {
13229                break stream;
13230            }
13231            assert!(
13232                daemon.child.try_wait().unwrap().is_none(),
13233                "daemon failed before rendezvous"
13234            );
13235            assert!(
13236                std::time::Instant::now() < deadline,
13237                "daemon did not reach CreateProcess failpoint"
13238            );
13239            std::thread::yield_now();
13240        };
13241        stream
13242            .set_read_timeout(Some(Duration::from_secs(30)))
13243            .unwrap();
13244        let mut pid = [0; 4];
13245        stream.read_exact(&mut pid).unwrap();
13246        let pid = u32::from_ne_bytes(pid);
13247        assert!(subc_jobobject::process_exists(pid));
13248        // Terminate only the daemon, not its outer job: otherwise the host job
13249        // would also kill a child whose daemon never joined its root job.
13250        daemon.child.kill().unwrap();
13251        daemon.child.wait().unwrap();
13252        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
13253        if !exited {
13254            daemon.job.terminate().unwrap();
13255        }
13256        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
13257    }
13258
13259    #[cfg(windows)]
13260    #[tokio::test]
13261    async fn windows_containment_failure_never_resumes() {
13262        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
13263        let sentinel = home.join("store-touched");
13264        let mut spec = spec();
13265        spec.env
13266            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
13267        spec.env.push((
13268            "SUBC_TEST_STORE_TOUCH".into(),
13269            sentinel.display().to_string(),
13270        ));
13271        let supervisor =
13272            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13273        let result = supervisor.spawn(spec);
13274        // A version that resumes despite assignment failure must have time to
13275        // execute its sentinel before dropping the returned module kills it.
13276        if let Ok(module) = result {
13277            let _ = tokio::time::timeout(Duration::from_secs(10), async {
13278                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
13279                    tokio::task::yield_now().await;
13280                }
13281            })
13282            .await;
13283            let _ = module.stop().await;
13284            panic!(
13285                "assignment failure resumed a module; store touched={}",
13286                sentinel.exists()
13287            );
13288        }
13289        assert!(
13290            !sentinel.exists(),
13291            "refused module executed store-touch sentinel"
13292        );
13293        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
13294    }
13295
13296    #[cfg(windows)]
13297    #[tokio::test]
13298    async fn windows_nested_daemon_fixture() {
13299        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
13300            return;
13301        };
13302        let mut spec = spec();
13303        spec.env.push((
13304            "SUBC_TEST_STORE_TOUCH".into(),
13305            sentinel.to_str().unwrap().into(),
13306        ));
13307        let supervisor =
13308            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13309        let module = supervisor
13310            .spawn(spec)
13311            .expect("per-module job must nest below daemon and host jobs");
13312        tokio::time::timeout(Duration::from_secs(10), async {
13313            while !PathBuf::from(&sentinel).exists() {
13314                tokio::task::yield_now().await;
13315            }
13316        })
13317        .await
13318        .unwrap();
13319        module.stop().await.unwrap();
13320    }
13321
13322    #[cfg(windows)]
13323    #[test]
13324    fn windows_module_job_nests_under_host_job() {
13325        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
13326        let sentinel = home.join("nested-store-touched");
13327        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13328        command
13329            .args([
13330                "--exact",
13331                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
13332                "--nocapture",
13333            ])
13334            .env("SUBC_TEST_NESTED_STORE", &sentinel);
13335        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13336        assert!(daemon.child.wait().unwrap().success());
13337        assert!(sentinel.exists(), "nested protocol:none module never ran");
13338    }
13339}
13340
13341#[cfg(target_os = "linux")]
13342pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
13343    use subc_cgroup::KillOutcome;
13344    match subc_cgroup::kill_module(placement, module_id) {
13345        KillOutcome::Killed => {}
13346        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
13347            debug!(
13348                module_id,
13349                "cgroup tree kill unavailable; using direct-child kill"
13350            );
13351        }
13352        KillOutcome::IoError { path, error } => {
13353            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
13354        }
13355    }
13356}
13357
13358/// Assign a freshly spawned Windows child without ever resuming on failure.
13359///
13360/// Steps 2 and 3 of the suspended-create contract: the job is created and the
13361/// child assigned **while it is still suspended** (step 1 is
13362/// `suspend_on_create_async` at the spawn site). The caller resumes only after
13363/// the child's roster admission is durable. Failed containment kills and reaps
13364/// the suspended child instead of running an unprotected module.
13365#[cfg(windows)]
13366fn contain_spawned_child(
13367    child: &mut Child,
13368    spec: &ModuleSpec,
13369    carried: Option<&Arc<containment::CarriedContainment>>,
13370) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
13371    let assignment = || {
13372        let job = match carried {
13373            Some(container) => container.job(),
13374            None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
13375        };
13376        #[cfg(test)]
13377        spawn_record_tests::assignment_failpoint(spec)?;
13378        job.lock()
13379            .unwrap_or_else(|p| p.into_inner())
13380            .assign(child)?;
13381        Ok::<_, io::Error>(job)
13382    };
13383    match assignment() {
13384        Ok(job) => Ok(Some(job)),
13385        Err(source) => {
13386            kill_and_reap_rejected_child(child);
13387            Err(SuperviseError::Spawn {
13388                program: spec.program.clone(),
13389                source: io::Error::other(format!(
13390                    "Windows module job containment failed: {source}"
13391                )),
13392                cgroup_path: None,
13393            })
13394        }
13395    }
13396}
13397
13398/// Rejection is synchronous: do not return a spawn error while the child can
13399/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
13400fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
13401    #[cfg(unix)]
13402    group.signal_rejected(child);
13403    let _ = child.start_kill();
13404    while matches!(child.try_wait(), Ok(None)) {
13405        std::thread::yield_now();
13406    }
13407}
13408
13409/// Resume a suspended child, killing it if it cannot be started.
13410///
13411/// A suspended process holds a pid and does nothing, so there is no useful
13412/// state to return: the caller gets an error and the spawn fails.
13413#[cfg(windows)]
13414fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
13415    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
13416        // Kill it here rather than leaving a suspended process for the caller
13417        // to notice; `kill_on_drop` would eventually do this, but the module
13418        // would have been reported as running in between.
13419        let _ = subc_os::child_spawn::status(
13420            std::process::Command::new("taskkill.exe")
13421                .args(["/PID", &pid.to_string(), "/T", "/F"])
13422                .stdin(Stdio::null())
13423                .stdout(Stdio::null())
13424                .stderr(Stdio::null()),
13425        );
13426        return Err(SuperviseError::Spawn {
13427            program: spec.program.clone(),
13428            source,
13429            cgroup_path: None,
13430        });
13431    }
13432    Ok(())
13433}
13434
13435#[cfg(target_os = "linux")]
13436fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13437    match placement.remove_module(module_id) {
13438        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
13439        Err(error) => warn!(
13440            module_id,
13441            error = %error,
13442            "could not remove module cgroup after process exit; continuing teardown"
13443        ),
13444    }
13445}
13446
13447#[cfg(target_os = "linux")]
13448const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
13449#[cfg(target_os = "linux")]
13450const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
13451
13452#[cfg(target_os = "linux")]
13453async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13454    let path = placement.module_path(module_id).ok();
13455    if path.as_ref().is_some_and(|path| {
13456        std::fs::read_to_string(path.join("cgroup.events"))
13457            .ok()
13458            .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
13459    }) {
13460        remove_module_cgroup(placement, module_id);
13461        return;
13462    }
13463
13464    // Freeze before enumerating: a frozen task cannot exit and release its PID
13465    // for reuse by a process outside this module while SIGTERM is being sent.
13466    if terminate_reaped_cgroup(placement, module_id).await {
13467        if let Some(path) = &path {
13468            let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
13469            while cgroup_is_populated(path) && Instant::now() < deadline {
13470                sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
13471            }
13472        }
13473    }
13474
13475    // Reaping the direct child is not proof its descendants exited. Kill any
13476    // survivors; when `cgroup.kill` succeeds, wait for the kernel to report the
13477    // cgroup empty before rmdir.
13478    if matches!(
13479        subc_cgroup::kill_module(Some(placement), module_id),
13480        subc_cgroup::KillOutcome::Killed
13481    ) {
13482        if let Some(path) = &path {
13483            while cgroup_is_populated(path) {
13484                sleep(Duration::from_millis(1)).await;
13485            }
13486        }
13487    }
13488    remove_module_cgroup(placement, module_id);
13489}
13490
13491#[cfg(target_os = "linux")]
13492fn cgroup_is_populated(path: &std::path::Path) -> bool {
13493    std::fs::read_to_string(path.join("cgroup.events"))
13494        .ok()
13495        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
13496}
13497
13498#[cfg(target_os = "linux")]
13499async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
13500    let frozen =
13501        match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
13502            .await
13503        {
13504            subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
13505            outcome => {
13506                tracing::debug!(
13507                    module_id,
13508                    ?outcome,
13509                    "cgroup freeze failed; skipping SIGTERM grace"
13510                );
13511                return false;
13512            }
13513        };
13514    let listed = match frozen.processes() {
13515        subc_cgroup::ProcsOutcome::Listed(pids) => {
13516            for pid in pids.into_iter().filter(|pid| *pid > 1) {
13517                if let Some(pid) = rustix::process::Pid::from_raw(pid) {
13518                    if let Err(error) =
13519                        rustix::process::kill_process(pid, rustix::process::Signal::TERM)
13520                    {
13521                        tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
13522                    }
13523                }
13524            }
13525            true
13526        }
13527        outcome => {
13528            tracing::debug!(
13529                module_id,
13530                ?outcome,
13531                "cgroup process listing failed; skipping SIGTERM grace"
13532            );
13533            false
13534        }
13535    };
13536    match frozen.thaw() {
13537        subc_cgroup::ThawOutcome::Thawed => listed,
13538        outcome => {
13539            tracing::debug!(
13540                module_id,
13541                ?outcome,
13542                "cgroup thaw failed; proceeding to cgroup.kill"
13543            );
13544            false
13545        }
13546    }
13547}
13548
13549#[cfg(target_os = "linux")]
13550fn apply_cgroup_placement(
13551    command: &mut Command,
13552    spec: &ModuleSpec,
13553    path: &std::path::Path,
13554) -> Result<(), SuperviseError> {
13555    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
13556        module_id: spec.module_id.clone(),
13557        source,
13558    })
13559}
13560
13561fn capture_retention(spec: &ModuleSpec) -> Retention {
13562    let defaults = Retention::default();
13563    let value = |name: &str| {
13564        spec.env
13565            .iter()
13566            .rev()
13567            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
13568    };
13569    Retention {
13570        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
13571            .and_then(|value| value.parse().ok())
13572            .unwrap_or(defaults.max_file_mb),
13573        keep: value(CAPTURE_KEEP_ENV)
13574            .and_then(|value| value.parse().ok())
13575            .unwrap_or(defaults.keep),
13576        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
13577            .and_then(|value| value.parse().ok())
13578            .unwrap_or(defaults.max_age_days),
13579    }
13580}
13581
13582/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
13583/// module's registration to the exact process the supervisor spawned.
13584fn generate_launch_nonce() -> Result<String, SuperviseError> {
13585    let mut bytes = [0u8; 32];
13586    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
13587        reason: source.to_string(),
13588    })?;
13589    let mut hex = String::with_capacity(64);
13590    for b in bytes {
13591        use std::fmt::Write;
13592        let _ = write!(hex, "{b:02x}");
13593    }
13594    Ok(hex)
13595}
13596
13597/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
13598/// signal about how many leading bytes matched.
13599fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
13600    if a.len() != b.len() {
13601        return false;
13602    }
13603    let mut diff = 0u8;
13604    for (x, y) in a.iter().zip(b.iter()) {
13605        diff |= x ^ y;
13606    }
13607    diff == 0
13608}
13609
13610/// The kernel's image after an acknowledged exec, shared by ordinary launches
13611/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
13612/// not identities inferred from a configured pathname.
13613#[cfg(not(windows))]
13614fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
13615    let process = subc_os::Process::open(pid).ok().flatten()?;
13616    #[cfg(target_os = "linux")]
13617    return sample_after_linux_exec(
13618        || {
13619            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
13620            let committed = linux_exec_committed(&stat);
13621            #[cfg(test)]
13622            spawn_image_tests::report_exec_state(committed);
13623            committed
13624        },
13625        || process.observe(),
13626    );
13627    #[cfg(not(target_os = "linux"))]
13628    process.observe()
13629}
13630
13631/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
13632/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
13633/// installation. Until it clears, /proc/pid/exe can still name the daemon.
13634#[cfg(target_os = "linux")]
13635fn linux_exec_committed(stat: &str) -> Option<bool> {
13636    const PF_FORKNOEXEC: u64 = 0x40;
13637    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
13638    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
13639}
13640
13641#[cfg(target_os = "linux")]
13642fn sample_after_linux_exec(
13643    mut exec_committed: impl FnMut() -> Option<bool>,
13644    observe: impl FnOnce() -> Option<subc_os::Observation>,
13645) -> Option<subc_os::Observation> {
13646    let deadline = std::time::Instant::now() + Duration::from_secs(1);
13647    while !exec_committed()? {
13648        // A failed or indefinitely stalled exec must not publish the inherited
13649        // image as authority to signal a future orphan. Missing identity keeps
13650        // the existing conservative orphan-sweep behavior.
13651        if std::time::Instant::now() >= deadline {
13652            return None;
13653        }
13654        std::thread::sleep(Duration::from_micros(50));
13655    }
13656    observe()
13657}
13658
13659#[cfg(all(test, target_os = "linux"))]
13660mod spawn_image_tests {
13661    use super::*;
13662    use std::cell::{Cell, RefCell};
13663
13664    enum ExecEvent {
13665        Uncommitted,
13666        Image(Option<subc_os::Observation>),
13667    }
13668
13669    thread_local! {
13670        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
13671    }
13672
13673    pub(super) fn report_exec_state(committed: Option<bool>) {
13674        if committed == Some(false) {
13675            EXEC_EVENTS.with(|events| {
13676                if let Some(events) = events.borrow_mut().take() {
13677                    let _ = events.send(ExecEvent::Uncommitted);
13678                }
13679            });
13680        }
13681    }
13682
13683    #[test]
13684    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
13685        use std::io::{Read, Write};
13686        use std::os::fd::AsRawFd;
13687
13688        let descriptor = std::fs::File::open("/dev/null").unwrap();
13689        let mut command = std::process::Command::new("/bin/sleep");
13690        command.arg("30");
13691        let (mut ready, mut release) =
13692            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
13693                .unwrap();
13694        let spawn = std::thread::spawn(move || command.spawn());
13695        let mut message = [0u8; 8];
13696        ready.read_exact(&mut message).unwrap();
13697        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
13698        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
13699        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
13700        let (events, received) = std::sync::mpsc::channel();
13701        let observer = std::thread::spawn(move || {
13702            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
13703            let image = observe_spawned_image(pid);
13704            let _ = events.send(ExecEvent::Image(image));
13705        });
13706        // Release only once the production observer has seen the uncommitted
13707        // exec, or has incorrectly returned the inherited image. Either way,
13708        // release and reap before asserting so a regression cannot leak a child.
13709        let first = received.recv_timeout(Duration::from_secs(5));
13710        release.write_all(&[1]).unwrap();
13711        let mut child = spawn.join().unwrap().unwrap();
13712        let after = match first {
13713            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
13714                Ok(ExecEvent::Image(image)) => image,
13715                _ => None,
13716            },
13717            Ok(ExecEvent::Image(image)) => image,
13718            Err(_) => None,
13719        };
13720        observer.join().unwrap();
13721        child.kill().unwrap();
13722        child.wait().unwrap();
13723
13724        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
13725        assert_eq!(
13726            inherited.unwrap().executable,
13727            subc_os::file_identity(&std::env::current_exe().unwrap())
13728        );
13729        assert_eq!(
13730            after.unwrap().executable,
13731            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
13732        );
13733    }
13734
13735    #[test]
13736    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
13737        let committed = Cell::new(false);
13738        let inherited = subc_os::Observation {
13739            start_time: 23,
13740            executable: Some(subc_os::FileIdentity {
13741                device: 29,
13742                inode: 31,
13743            }),
13744        };
13745        let module = subc_os::Observation {
13746            start_time: 23,
13747            executable: Some(subc_os::FileIdentity {
13748                device: 29,
13749                inode: 37,
13750            }),
13751        };
13752        let polls = Cell::new(0);
13753        let image = sample_after_linux_exec(
13754            || {
13755                polls.set(polls.get() + 1);
13756                // Hold the inherited image through the first poll, as in the
13757                // posix_spawn parent-wake window, then commit the module image.
13758                committed.set(polls.get() == 2);
13759                Some(committed.get())
13760            },
13761            || Some(if committed.get() { module } else { inherited }),
13762        );
13763        assert_eq!(image, Some(module));
13764        assert_eq!(polls.get(), 2);
13765    }
13766
13767    #[test]
13768    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
13769        assert_eq!(
13770            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
13771            Some(false)
13772        );
13773        assert_eq!(
13774            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
13775            Some(true)
13776        );
13777        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
13778        assert_eq!(linux_exec_committed("17 (a) R"), None);
13779    }
13780
13781    #[test]
13782    fn unreadable_exec_state_never_samples_an_inherited_image() {
13783        assert_eq!(
13784            sample_after_linux_exec(
13785                || None,
13786                || panic!("must not sample before exec confirmation")
13787            ),
13788            None
13789        );
13790    }
13791
13792    #[test]
13793    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
13794        assert_eq!(
13795            sample_after_linux_exec(
13796                || Some(false),
13797                || panic!("must not sample after an exec timeout")
13798            ),
13799            None
13800        );
13801    }
13802}
13803
13804/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
13805/// different executable after the no-delete capture was taken.
13806#[cfg(windows)]
13807pub(crate) fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
13808    if let Err(error) = require_program_search_path(spec) {
13809        return Err(match error {
13810            SuperviseError::Spawn { source, .. } => source,
13811            error => io::Error::other(error),
13812        });
13813    }
13814    let mut program = spec.program.clone();
13815    if program.is_absolute() || program.components().count() > 1 {
13816        // An explicit path can name a PE image with no extension. Try that
13817        // exact file first, as Command does, before falling back to .exe.
13818        match subc_os::ExecutableCapture::open(&program) {
13819            Ok(capture) => return Ok(capture),
13820            Err(error)
13821                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
13822            Err(error) => return Err(error),
13823        }
13824        program.set_extension("exe");
13825        return subc_os::ExecutableCapture::open(&program);
13826    }
13827    if program.extension().is_none() {
13828        program.set_extension("exe");
13829    }
13830    let custom_path = spec
13831        .env
13832        .iter()
13833        .rev()
13834        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
13835    let mut directories = Vec::new();
13836    // These are Windows' default executable search locations. A configured
13837    // PATH still overrides the inherited PATH, not the system directories.
13838    if let Some(parent) = std::env::current_exe()?.parent() {
13839        directories.push(parent.to_path_buf());
13840    }
13841    directories.push(std::env::current_dir()?);
13842    if let Some(windows) = std::env::var_os("SystemRoot") {
13843        let windows = PathBuf::from(windows);
13844        directories.push(windows.join("System32"));
13845        directories.push(windows.clone());
13846    }
13847    let path = custom_path
13848        .map(|(_, value)| std::ffi::OsString::from(value))
13849        .or_else(|| std::env::var_os("PATH"));
13850    if let Some(path) = path {
13851        directories.extend(std::env::split_paths(&path));
13852    }
13853    for directory in directories {
13854        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
13855            Ok(capture) => return Ok(capture),
13856            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
13857            Err(error) => return Err(error),
13858        }
13859    }
13860    Err(io::Error::new(
13861        io::ErrorKind::NotFound,
13862        "resolved executable not found",
13863    ))
13864}
13865
13866async fn spawn_and_mark_running_async(
13867    spec: &ModuleSpec,
13868    runtime: &SupervisorRuntimeConfig,
13869    snapshot: &SharedSnapshot,
13870) -> Result<SupervisedChild, SuperviseError> {
13871    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13872    let registration_watch = watch_registration(&runtime.registry, spec);
13873    #[cfg(unix)]
13874    let joining_group = containment::current(&runtime.child_roster.containment)
13875        .is_some_and(|container| container.group().is_some());
13876    let result = spawn_child_async(spec, runtime).await;
13877    #[cfg(unix)]
13878    let result = if result.is_err()
13879        && joining_group
13880        && !runtime.child_roster.is_closed()
13881        && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13882        && runtime
13883            .child_roster
13884            .previous_instance(&spec.module_id)
13885            .is_none()
13886    {
13887        // A failed group join has already ended the old container. Its single
13888        // fresh-container retry belongs to the same restart budget entry.
13889        warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
13890        spawn_child_async(spec, runtime).await
13891    } else {
13892        result
13893    };
13894    let mut child = result?;
13895    arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
13896    if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
13897        containment::terminate_blocking(&runtime.child_roster.containment);
13898        return Err(error);
13899    }
13900    if let Some(spawn) = activity_spawn {
13901        spawn.complete(snapshot);
13902    }
13903    Ok(child)
13904}
13905
13906#[cfg(test)]
13907fn spawn_and_mark_running(
13908    spec: &ModuleSpec,
13909    runtime: &SupervisorRuntimeConfig,
13910    snapshot: &SharedSnapshot,
13911) -> Result<SupervisedChild, SuperviseError> {
13912    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13913    let registration_watch = watch_registration(&runtime.registry, spec);
13914    #[cfg(unix)]
13915    let joining_group = containment::current(&runtime.child_roster.containment)
13916        .is_some_and(|container| container.group().is_some());
13917    let spawn = || {
13918        spawn_child(
13919            spec,
13920            runtime.connection_file_path.as_deref(),
13921            runtime.supervisor_handle.as_ref(),
13922            &runtime.stderr_ring,
13923            runtime.capture_logs_dir.as_deref(),
13924            &runtime.child_roster,
13925            runtime.spawn_delivery(),
13926            #[cfg(target_os = "linux")]
13927            runtime.cgroup_placement.as_ref(),
13928            runtime.register_timeout,
13929        )
13930    };
13931    let result = spawn();
13932    #[cfg(unix)]
13933    let result = if result.is_err()
13934        && joining_group
13935        && !runtime.child_roster.is_closed()
13936        && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13937        && runtime
13938            .child_roster
13939            .previous_instance(&spec.module_id)
13940            .is_none()
13941    {
13942        // The failed join already destroyed and verified the old container.
13943        // Try one fresh group as part of this same restart, without spending
13944        // another crash-budget entry. Cgroups and Windows jobs do not retry.
13945        warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
13946        spawn()
13947    } else {
13948        result
13949    };
13950    let child = result?;
13951    let mut child = child;
13952    arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
13953    if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
13954        containment::terminate_blocking(&runtime.child_roster.containment);
13955        return Err(error);
13956    }
13957    if let Some(spawn) = activity_spawn {
13958        spawn.complete(snapshot);
13959    }
13960    Ok(child)
13961}
13962
13963struct RegistrationWatch {
13964    changes: watch::Receiver<u64>,
13965    baseline: u64,
13966}
13967
13968// Capture before spawning: a fast child's accepted HELLO can disappear from the
13969// active registry before the monitor first runs. A pre-existing registration
13970// must not satisfy a new process's deadline either.
13971fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
13972    if spec.protocol != ModuleProtocol::Subc {
13973        return None;
13974    }
13975    let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
13976    let baseline = *changes.borrow();
13977    Some(RegistrationWatch { changes, baseline })
13978}
13979
13980fn arm_registration_deadline(
13981    child: &mut SupervisedChild,
13982    budget: Duration,
13983    registration_watch: Option<RegistrationWatch>,
13984) {
13985    child.registration_watch = registration_watch;
13986    child.registration_deadline = (child.protocol == ModuleProtocol::Subc
13987        && child.awaiting_registration())
13988    .then_some(child.spawned_at + budget);
13989}
13990
13991async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
13992    child.retire_registration();
13993    if let Some(container) = child.child.container() {
13994        container.stop_members().await;
13995        child.kill_leader()?;
13996        child.wait().await?;
13997        return Ok(());
13998    }
13999    #[cfg(unix)]
14000    {
14001        // Children lead their own process groups. Signal the whole group so a
14002        // helper stuck before HELLO cannot survive its supervised parent.
14003        let group = unreaped_registration_group(child)?;
14004        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
14005        // Process-group safety rule: only signal a group while its leader is
14006        // our unreaped child. Even a zombie leader reserves its pid/group id;
14007        // reaping it before KILL would let that id name an unrelated group.
14008        // Sleep without waiting so TERM-ignoring helpers get the full grace.
14009        sleep(crate::child_roster::TERM_TO_KILL).await;
14010        let group = unreaped_registration_group(child)?;
14011        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
14012    }
14013    // Windows terminates the containing Job Object as well as the direct
14014    // child. Linux also kills the child's cgroup when placement is enabled.
14015    child.start_kill()?;
14016    child.wait().await?;
14017    Ok(())
14018}
14019
14020#[cfg(unix)]
14021fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
14022    if let Some(container) = child.child.container() {
14023        return container
14024            .group()
14025            .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
14026            .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
14027    }
14028    // The group id comes only from the pid recorded at spawn, when the child
14029    // created its own group. Never admit init's group or a negative group id.
14030    let group = i32::try_from(child.pid)
14031        .ok()
14032        .filter(|pid| *pid > 1)
14033        .and_then(rustix::process::Pid::from_raw)
14034        .ok_or_else(|| {
14035            io::Error::new(
14036                io::ErrorKind::InvalidInput,
14037                "invalid supervised process group",
14038            )
14039        })?;
14040    if child.child.id() != Some(child.pid) {
14041        return Err(io::Error::new(
14042            io::ErrorKind::InvalidInput,
14043            "refusing process-group signal after supervised child was reaped",
14044        ));
14045    }
14046    Ok(group)
14047}
14048
14049enum RegistrationWaitOutcome {
14050    Registered,
14051    Exited(ExitReport),
14052    TimedOut,
14053}
14054
14055struct ReloadRegistrationFailure {
14056    exit_report: ExitReport,
14057    reason: String,
14058}
14059
14060#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14061enum BusyGaugeObservation {
14062    Quiescent,
14063    Busy,
14064    Omitted,
14065}
14066
14067/// What one health answer said about one declared busy gauge.
14068#[derive(Debug, Clone, PartialEq, Eq)]
14069enum GaugeVerdict {
14070    /// The gauge read 0: this gauge is not what holds the drain.
14071    Zero,
14072    /// The gauge read a positive count.
14073    NonZero(u64),
14074    /// The gauge was present but not a non-negative integer (`0.0`, a string,
14075    /// a negative number). It counts as busy, because the daemon cannot tell
14076    /// it is idle. Carries the value as JSON text, cut to a short prefix.
14077    NotACount(String),
14078    /// The health answer had no metrics object, or one without this gauge.
14079    Missing,
14080    /// The probe produced no health answer to read; carries the probe error's
14081    /// evidence label (`no-answer` for a probe that timed out).
14082    ProbeFailed(&'static str),
14083}
14084
14085impl GaugeVerdict {
14086    fn reads_quiet(&self) -> bool {
14087        matches!(self, Self::Zero)
14088    }
14089}
14090
14091impl fmt::Display for GaugeVerdict {
14092    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
14093        match self {
14094            Self::Zero => f.write_str("zero"),
14095            Self::NonZero(value) => write!(f, "non_zero({value})"),
14096            Self::NotACount(value) => write!(f, "not_a_count({value})"),
14097            Self::Missing => f.write_str("missing_from_health_body"),
14098            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
14099        }
14100    }
14101}
14102
14103/// The longest JSON text kept for a gauge value that is not a count, so a
14104/// module reporting a large object as a gauge cannot flood the drain log.
14105const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
14106
14107/// Read each declared gauge out of one health answer's metrics.
14108fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
14109    let metrics = metrics.and_then(Value::as_object);
14110    gauges
14111        .iter()
14112        .map(
14113            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
14114                None => GaugeVerdict::Missing,
14115                Some(value) => match value.as_u64() {
14116                    Some(0) => GaugeVerdict::Zero,
14117                    Some(count) => GaugeVerdict::NonZero(count),
14118                    None => {
14119                        let mut text = value.to_string();
14120                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
14121                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
14122                            while !text.is_char_boundary(end) {
14123                                end -= 1;
14124                            }
14125                            text.truncate(end);
14126                            text.push_str("...");
14127                        }
14128                        GaugeVerdict::NotACount(text)
14129                    }
14130                },
14131            },
14132        )
14133        .collect()
14134}
14135
14136/// The drain's overall reading of one health answer: omitted when any
14137/// declared gauge is missing, quiescent only when every gauge reads 0.
14138fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
14139    if verdicts
14140        .iter()
14141        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
14142    {
14143        BusyGaugeObservation::Omitted
14144    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
14145        BusyGaugeObservation::Quiescent
14146    } else {
14147        BusyGaugeObservation::Busy
14148    }
14149}
14150
14151/// One declared gauge's history over a drain: the latest verdict on it, and
14152/// the latest value actually read for it (which a later failed probe or
14153/// missing gauge does not erase). Times are offsets from the start of the
14154/// drain's wait.
14155#[derive(Debug, Clone, Default, PartialEq, Eq)]
14156struct GaugeTrace {
14157    latest: Option<(GaugeVerdict, Duration)>,
14158    last_value: Option<(GaugeVerdict, Duration)>,
14159}
14160
14161/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
14162/// its ceiling can say in its timeout line which declared gauge held it and
14163/// why. Without it, a gauge that read busy, a gauge the module never
14164/// reported, and a probe that never got an answer all looked the same: a
14165/// drain that ended at its ceiling with nothing in flight.
14166#[derive(Debug, Clone, Default, PartialEq, Eq)]
14167struct DrainGaugeTrace {
14168    /// Health probes the drain sent.
14169    probes: u32,
14170    /// Each declared gauge, in declaration order.
14171    gauges: Vec<(String, GaugeTrace)>,
14172}
14173
14174impl DrainGaugeTrace {
14175    fn new(gauges: &[String]) -> Self {
14176        Self {
14177            probes: 0,
14178            gauges: gauges
14179                .iter()
14180                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
14181                .collect(),
14182        }
14183    }
14184
14185    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
14186    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
14187        self.probes = self.probes.saturating_add(1);
14188        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
14189            trace.latest = Some((verdict.clone(), at));
14190            if matches!(
14191                verdict,
14192                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
14193            ) {
14194                trace.last_value = Some((verdict.clone(), at));
14195            }
14196        }
14197    }
14198}
14199
14200/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
14201/// declared gauge, separated by ` | `. A value is shown as its verdict
14202/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
14203/// and whether that counted as busy.
14204impl fmt::Display for DrainGaugeTrace {
14205    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
14206        if self.gauges.is_empty() {
14207            return f.write_str("none declared");
14208        }
14209        for (index, (name, trace)) in self.gauges.iter().enumerate() {
14210            if index > 0 {
14211                f.write_str(" | ")?;
14212            }
14213            write!(f, "{name}: ")?;
14214            match &trace.latest {
14215                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
14216                None => f.write_str("never probed")?,
14217            }
14218            match &trace.last_value {
14219                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
14220                None => f.write_str("; no value ever read")?,
14221            }
14222        }
14223        Ok(())
14224    }
14225}
14226
14227pub(crate) fn declared_busy_gauges(
14228    registry: &Registry,
14229    module_id: &str,
14230) -> Result<Vec<String>, SuperviseError> {
14231    busy_gauges_of(
14232        registry
14233            .get_module(module_id)
14234            .map_err(SuperviseError::Registry)?,
14235    )
14236}
14237
14238/// [`declared_busy_gauges`] for the registration a connection holds, in any
14239/// slot: after cutover the incumbent is no longer the id's active
14240/// registration, and its own manifest is the one that names its gauges.
14241fn declared_busy_gauges_for_connection(
14242    registry: &Registry,
14243    connection_id: ConnectionId,
14244) -> Result<Vec<String>, SuperviseError> {
14245    busy_gauges_of(
14246        registry
14247            .get_module_by_connection(connection_id)
14248            .map_err(SuperviseError::Registry)?,
14249    )
14250}
14251
14252fn busy_gauges_of(
14253    registration: Option<crate::registry::ModuleRegistration>,
14254) -> Result<Vec<String>, SuperviseError> {
14255    let Some(registration) = registration else {
14256        return Ok(Vec::new());
14257    };
14258    let Some(self_signals) = registration.manifest.self_signals else {
14259        return Ok(Vec::new());
14260    };
14261
14262    let mut gauges = Vec::new();
14263    for declaration in self_signals {
14264        if declaration.kind != SelfSignalKind::Busy {
14265            continue;
14266        }
14267        match declaration.anchored_to {
14268            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
14269                gauges.extend(declared)
14270            }
14271            _ => {
14272                // An invalid Busy anchor is fail-safe: the empty name cannot be
14273                // present in a conforming health report, so this drain stays busy.
14274                gauges.push(String::new());
14275            }
14276        }
14277    }
14278    Ok(gauges)
14279}
14280
14281/// Wait for `endpoint` to have nothing in flight and, when the module declares
14282/// busy gauges, for a health probe to report them quiet. The probe is addressed
14283/// by `scope`: a swap's superseded incumbent must be asked about its own
14284/// gauges, and by module id the probe would reach the promoted candidate.
14285/// While the gauges read busy, the probe repeats every
14286/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
14287/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
14288/// timeout line.
14289#[allow(clippy::too_many_arguments)]
14290async fn wait_for_forwarding_quiescence(
14291    forwarding: &ForwardingTable,
14292    module_id: &str,
14293    runtime: &SupervisorRuntimeConfig,
14294    endpoint: crate::ModuleEndpointId,
14295    deadline: Instant,
14296    busy_gauges: &[String],
14297    scope: DrainScope,
14298    trace: &mut DrainGaugeTrace,
14299) -> Result<bool, SuperviseError> {
14300    let wait_started = Instant::now();
14301    let mut gauges_quiescent = busy_gauges.is_empty();
14302    let mut next_probe_at = wait_started;
14303    let mut omission_reported = false;
14304
14305    loop {
14306        let now = Instant::now();
14307        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
14308            let report = match scope {
14309                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
14310                DrainScope::Endpoint(endpoint) => {
14311                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
14312                }
14313            };
14314            let verdicts = match &report {
14315                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
14316                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
14317            };
14318            trace.record(&verdicts, wait_started.elapsed());
14319            gauges_quiescent = match report {
14320                Ok(_) => match busy_gauge_observation(&verdicts) {
14321                    BusyGaugeObservation::Quiescent => true,
14322                    BusyGaugeObservation::Busy => false,
14323                    BusyGaugeObservation::Omitted => {
14324                        // Once per drain: a module that never reports a gauge
14325                        // it declared would otherwise say so on every probe.
14326                        if !omission_reported {
14327                            forwarding
14328                                .counters()
14329                                .increment_drains_with_undeclared_gauge();
14330                            let missing = busy_gauges
14331                                .iter()
14332                                .zip(&verdicts)
14333                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
14334                                .map(|(gauge, _)| gauge.as_str())
14335                                .collect::<Vec<_>>()
14336                                .join(",");
14337                            warn!(
14338                                module_id,
14339                                missing_gauges = %missing,
14340                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
14341                            );
14342                            omission_reported = true;
14343                        }
14344                        false
14345                    }
14346                },
14347                Err(err) => {
14348                    warn!(
14349                        module_id,
14350                        error = %err,
14351                        "drain health.check did not produce declared busy gauges; treating module as busy"
14352                    );
14353                    false
14354                }
14355            };
14356            // Counted from the probe's completion: the probe above was awaited,
14357            // so no second probe can start while one is still outstanding.
14358            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
14359        }
14360
14361        let in_flight = forwarding
14362            .endpoint_in_flight_count(endpoint)
14363            .map_err(SuperviseError::Forwarding)?;
14364        if in_flight == 0 && gauges_quiescent {
14365            return Ok(true);
14366        }
14367
14368        let now = Instant::now();
14369        if now >= deadline {
14370            return Ok(false);
14371        }
14372        let mut wait = deadline
14373            .saturating_duration_since(now)
14374            .min(REGISTRY_RELEASE_POLL);
14375        if !busy_gauges.is_empty() {
14376            wait = wait.min(next_probe_at.saturating_duration_since(now));
14377        }
14378        sleep(wait).await;
14379    }
14380}
14381
14382/// The wait between drain busy-gauge probes: the larger of
14383/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
14384/// drain's own in-flight poll interval.
14385fn drain_busy_reprobe_interval() -> Duration {
14386    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
14387}
14388
14389/// Logs why a drain gave up: the drain waited its full timeout without the
14390/// module going quiet, and teardown is being forced.
14391///
14392/// The line names two kinds of evidence, because a bare "timed out" hides
14393/// causes that need different fixes:
14394/// - `holdouts`: the requests and routes still open when the timeout hit,
14395///   and on which connections. One stuck call is a module bug; a held stream
14396///   on every session is a module that should end its streams when it
14397///   receives `module.draining`.
14398/// - `gauges`: for each busy gauge the module declares, its latest reading
14399///   and how many probes ran. This separates a module that kept reporting
14400///   itself busy, one whose health answer never contained the gauge, and one
14401///   that never answered the probes, which otherwise all look like a timeout
14402///   with nothing in flight.
14403fn warn_drain_timed_out(
14404    module_id: &str,
14405    drain_timeout: Duration,
14406    reason: RouteCloseReason,
14407    holdouts: &crate::forwarding::DrainHoldouts,
14408    gauges: &DrainGaugeTrace,
14409) {
14410    warn!(
14411        module_id = %module_id,
14412        waited = ?drain_timeout,
14413        ?reason,
14414        held_requests = holdouts.requests,
14415        held_routes = holdouts.routes,
14416        total_routes = holdouts.total_routes,
14417        top_connections = ?holdouts.top_connections,
14418        // `module_channel:corr`, so the module can find each held request
14419        // in its own log; capped, so `held_requests` is the full count.
14420        held = %holdouts
14421            .held
14422            .iter()
14423            .map(|(channel, corr)| format!("{channel}:{corr}"))
14424            .collect::<Vec<_>>()
14425            .join(","),
14426        gauge_probes = gauges.probes,
14427        busy_gauges = %gauges,
14428        "route drain timed out before request quiescence; forcing teardown"
14429    );
14430}
14431
14432/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
14433///
14434/// `Ok` is always honest and passed straight through -- the wait actually measured
14435/// in-flight state. `Err` means the wait produced no measurement at all (the
14436/// forwarding table's lock was poisoned), so `false` is reported as the one honest
14437/// constant: the drain did not complete. Never recomputed from route state, never a
14438/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
14439fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
14440    match wait_result {
14441        Ok(drained) => *drained,
14442        Err(_) => false,
14443    }
14444}
14445
14446fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
14447    for released in released_routes {
14448        let frame = match Frame::build_with_version(
14449            released.negotiated_ver,
14450            FrameType::Goodbye,
14451            control_flags(),
14452            released.channel,
14453            released.epoch,
14454            0,
14455            Vec::new(),
14456        ) {
14457            Ok(frame) => frame,
14458            Err(err) => {
14459                warn!(
14460                    route_channel = released.channel,
14461                    error = %err,
14462                    "failed to build supervisor drain route GOODBYE frame"
14463                );
14464                continue;
14465            }
14466        };
14467        if !released.close_on_delivery_failure() {
14468            crate::forwarding::send_module_route_goodbye(
14469                &forwarding.counters(),
14470                &released.sink,
14471                frame,
14472                released.module_id.as_deref(),
14473                "supervisor drain",
14474            );
14475            continue;
14476        }
14477        if let Err(err) = released.sink.try_send(frame) {
14478            warn!(
14479                target_connection_id = released.connection_id.get(),
14480                route_channel = released.channel,
14481                error = %err,
14482                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
14483            );
14484            let _ = forwarding.escalate_client_delivery_failure(
14485                released.connection_id,
14486                released.channel,
14487                released.epoch,
14488                CloseReason::new(
14489                    "route_goodbye_delivery_failed",
14490                    format!(
14491                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
14492                        released.channel
14493                    ),
14494                ),
14495                crate::forwarding::UndeliveredFrame {
14496                    module_id: released.module_id.as_deref(),
14497                    sink: &released.sink,
14498                },
14499            );
14500        }
14501    }
14502}
14503
14504async fn send_module_draining(
14505    module_id: &str,
14506    reason: RouteCloseReason,
14507    deadline_ms: u64,
14508    target: &ModuleDrainTarget,
14509    _wait_for_flush: bool,
14510) -> bool {
14511    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
14512        reason,
14513        deadline_ms,
14514    }) {
14515        Ok(body) => body,
14516        Err(err) => {
14517            warn!(
14518                module_id,
14519                error = %err,
14520                "failed to encode module draining command"
14521            );
14522            return false;
14523        }
14524    };
14525    let frame = match Frame::build_with_version(
14526        target.negotiated_ver,
14527        FrameType::Push,
14528        control_flags(),
14529        0,
14530        0,
14531        0,
14532        body,
14533    ) {
14534        Ok(frame) => frame,
14535        Err(err) => {
14536            warn!(
14537                module_id,
14538                error = %err,
14539                "failed to build module draining command frame"
14540            );
14541            return false;
14542        }
14543    };
14544    #[cfg(unix)]
14545    if _wait_for_flush {
14546        return target.sink.send_flushed(frame).await.is_ok();
14547    }
14548    if let Err(err) = target.sink.try_send(frame) {
14549        warn!(
14550            module_id,
14551            target_connection_id = target.endpoint.connection_id.get(),
14552            error = %err,
14553            "module draining command was not delivered to peer"
14554        );
14555        return false;
14556    }
14557    true
14558}
14559
14560/// The channel-0 GOODBYE that tells a module its stop is planned.
14561fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
14562    match Frame::build_with_version(
14563        negotiated_ver,
14564        FrameType::Goodbye,
14565        control_flags(),
14566        0,
14567        0,
14568        0,
14569        Vec::new(),
14570    ) {
14571        Ok(frame) => Some(frame),
14572        Err(err) => {
14573            warn!(
14574                module_id,
14575                error = %err,
14576                "failed to build module GOODBYE frame"
14577            );
14578            None
14579        }
14580    }
14581}
14582
14583/// Send every registered module connection its module GOODBYE at daemon
14584/// shutdown, then request that connection's close.
14585///
14586/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
14587/// before EOF, so the GOODBYE must reach the socket before the close. A close
14588/// request does not wait for the connection's queued frames: its writer gets a
14589/// bounded grace after the close, is aborted if it overruns it, and the daemon
14590/// process may exit before that grace ends. So with `wait_for_flush`, each
14591/// connection is closed only after its writer has acknowledged writing the
14592/// GOODBYE, or once a short shared budget runs out, so one module that is not
14593/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
14594/// are only queued, for a shutdown the operator has told to stop waiting.
14595/// A connection that is already gone is skipped.
14596#[cfg(unix)]
14597async fn send_module_goodbyes_for_daemon_shutdown(
14598    forwarding: &Arc<ForwardingTable>,
14599    reason: &CloseReason,
14600    wait_for_flush: bool,
14601) {
14602    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
14603    let targets = match forwarding.module_connections() {
14604        Ok(targets) => targets,
14605        Err(err) => {
14606            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
14607            return;
14608        }
14609    };
14610    let deadline = Instant::now() + GOODBYE_BUDGET;
14611    let mut sends = tokio::task::JoinSet::new();
14612    for target in targets {
14613        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
14614            continue;
14615        };
14616        if !wait_for_flush {
14617            if let Err(err) = target.sink.try_send(frame) {
14618                debug!(
14619                    module_id = %target.module_id,
14620                    error = %err,
14621                    "shutdown module GOODBYE was not queued"
14622                );
14623            }
14624            continue;
14625        }
14626        let forwarding = Arc::clone(forwarding);
14627        let reason = reason.clone();
14628        sends.spawn(async move {
14629            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
14630                Ok(Ok(())) => {}
14631                Ok(Err(err)) => debug!(
14632                    module_id = %target.module_id,
14633                    error = %err,
14634                    "module connection closed before its shutdown GOODBYE was written"
14635                ),
14636                Err(_) => warn!(
14637                    module_id = %target.module_id,
14638                    budget = ?GOODBYE_BUDGET,
14639                    "shutdown module GOODBYE was not written within its budget; closing anyway"
14640                ),
14641            }
14642            forwarding.request_connection_close(target.endpoint.connection_id, reason);
14643        });
14644    }
14645    // Every task ends by the shared deadline, so this wait is bounded too.
14646    while sends.join_next().await.is_some() {}
14647}
14648
14649async fn send_module_goodbye(
14650    module_id: &str,
14651    forwarding: &ForwardingTable,
14652    target: &ModuleDrainTarget,
14653    wait_for_flush: bool,
14654) {
14655    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
14656        return;
14657    };
14658    // A channel is shut down synchronously after its drain. Its final frame
14659    // must reach the socket writer first; queue admission alone can lose it.
14660    let sent = if wait_for_flush {
14661        timeout(Duration::from_millis(500), target.sink.send_flushed(frame))
14662            .await
14663            .map_err(|_| "module GOODBYE flush exceeded 500ms".to_owned())
14664            .and_then(|result| result.map_err(|err| err.to_string()))
14665    } else {
14666        target.sink.try_send(frame).map_err(|err| err.to_string())
14667    };
14668    if let Err(err) = sent {
14669        warn!(
14670            module_id,
14671            target_connection_id = target.endpoint.connection_id.get(),
14672            error = %err,
14673            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
14674        );
14675        forwarding.request_connection_close(
14676            target.endpoint.connection_id,
14677            CloseReason::new(
14678                "module_goodbye_delivery_failed",
14679                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
14680            ),
14681        );
14682    }
14683}
14684
14685#[derive(Clone, Copy)]
14686struct ForwardingDrainContext<'a> {
14687    spec: &'a ModuleSpec,
14688    runtime: &'a SupervisorRuntimeConfig,
14689    registry: &'a Registry,
14690    scope: DrainScope,
14691}
14692
14693/// Which process a forwarding drain addresses.
14694#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14695enum DrainScope {
14696    /// Whatever endpoint is active for the module id: every plain stop,
14697    /// restart and reload. Also moves the module's state to `Draining`.
14698    Active,
14699    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
14700    /// module id would resolve to the promoted candidate and leave neither
14701    /// process routable. The module's state is left alone, since the promoted
14702    /// candidate is what it describes and that process is running.
14703    Endpoint(crate::ModuleEndpointId),
14704}
14705
14706/// Whether a child being drained has already been asked to stop by the time
14707/// its drain wait starts.
14708///
14709/// The drain wait is the same budget whatever this says. What it decides is
14710/// whether the supervisor must ask by signal before that wait begins: a child
14711/// that nobody asked will sit out the whole budget and then be SIGKILLed,
14712/// healthy or not.
14713#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14714enum StopNotice {
14715    /// The module was sent `module.draining` and a module GOODBYE over its own
14716    /// registered connection, and stops itself.
14717    SentOverConnection,
14718    /// The forwarding drain found no registered connection for the module: a
14719    /// subc child spawned moments ago that has not sent HELLO yet, or a
14720    /// `protocol: "none"` child, which never registers.
14721    NoConnection,
14722    /// This path sends nothing over the module's connection: the supervisor has
14723    /// no forwarding table, or the caller stops the child without a forwarding
14724    /// drain.
14725    NotSent,
14726}
14727
14728async fn begin_forwarding_drain(
14729    spec: &ModuleSpec,
14730    runtime: &SupervisorRuntimeConfig,
14731    registry: &Registry,
14732    snapshot: &SharedSnapshot,
14733    enabled: Option<bool>,
14734    reason: RouteCloseReason,
14735) -> Result<StopNotice, SuperviseError> {
14736    let Some(forwarding) = runtime.forwarding.as_ref() else {
14737        return Err(SuperviseError::ReloadUnavailable {
14738            module_id: spec.module_id.clone(),
14739            reason: "supervisor was not configured with a forwarding table".to_string(),
14740        });
14741    };
14742
14743    begin_forwarding_drain_with(
14744        forwarding,
14745        ForwardingDrainContext {
14746            spec,
14747            runtime,
14748            registry,
14749            scope: DrainScope::Active,
14750        },
14751        snapshot,
14752        enabled,
14753        reason,
14754        runtime.drain_timeout,
14755    )
14756    .await
14757}
14758
14759async fn begin_forwarding_drain_if_configured(
14760    spec: &ModuleSpec,
14761    runtime: &SupervisorRuntimeConfig,
14762    registry: &Registry,
14763    snapshot: &SharedSnapshot,
14764    enabled: Option<bool>,
14765    reason: RouteCloseReason,
14766) -> Result<StopNotice, SuperviseError> {
14767    begin_forwarding_drain_with_timeout(
14768        spec,
14769        runtime,
14770        registry,
14771        snapshot,
14772        enabled,
14773        reason,
14774        runtime.drain_timeout,
14775    )
14776    .await
14777}
14778
14779/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
14780/// budget, for paths where the operator overrides the module's configured one
14781/// (`supervisor.restart{drain_timeout_ms}`).
14782async fn begin_forwarding_drain_with_timeout(
14783    spec: &ModuleSpec,
14784    runtime: &SupervisorRuntimeConfig,
14785    registry: &Registry,
14786    snapshot: &SharedSnapshot,
14787    enabled: Option<bool>,
14788    reason: RouteCloseReason,
14789    drain_timeout: Duration,
14790) -> Result<StopNotice, SuperviseError> {
14791    let Some(forwarding) = runtime.forwarding.as_ref() else {
14792        return Ok(StopNotice::NotSent);
14793    };
14794
14795    begin_forwarding_drain_with(
14796        forwarding,
14797        ForwardingDrainContext {
14798            spec,
14799            runtime,
14800            registry,
14801            scope: DrainScope::Active,
14802        },
14803        snapshot,
14804        enabled,
14805        reason,
14806        drain_timeout,
14807    )
14808    .await
14809}
14810
14811async fn begin_forwarding_drain_with(
14812    forwarding: &ForwardingTable,
14813    context: ForwardingDrainContext<'_>,
14814    snapshot: &SharedSnapshot,
14815    enabled: Option<bool>,
14816    reason: RouteCloseReason,
14817    drain_timeout: Duration,
14818) -> Result<StopNotice, SuperviseError> {
14819    let ForwardingDrainContext {
14820        spec,
14821        runtime,
14822        registry,
14823        scope,
14824    } = context;
14825    debug_assert_ne!(reason, RouteCloseReason::Crash);
14826    let terminal = matches!(reason, RouteCloseReason::Disable);
14827    let drain_started_at = Instant::now();
14828    let drain_deadline = drain_started_at + drain_timeout;
14829    let deadline_ms =
14830        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
14831    let busy_gauges = match scope {
14832        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
14833        DrainScope::Endpoint(endpoint) => {
14834            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
14835        }
14836    };
14837
14838    // Admission gate first: route.open/commit and route REQUEST admission are closed
14839    // before the first quiescence check, so the outstanding count can only fall.
14840    let gate_started = Instant::now();
14841    let drain_target = match scope {
14842        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
14843        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
14844    }
14845    .map_err(SuperviseError::Forwarding)?;
14846    // The instant admission closed, and how long taking the forwarding write
14847    // lock to close it took. The timeout line reports only the quiescence
14848    // wait, so without this a drain that started late looked like one that
14849    // started on time.
14850    info!(
14851        module_id = %spec.module_id,
14852        ?reason,
14853        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
14854        connected = drain_target.is_some(),
14855        "module drain began; route admission closed"
14856    );
14857    if scope == DrainScope::Active {
14858        update_snapshot(snapshot, Some(&spec.module_id), |state| {
14859            state.state = ModuleState::Draining;
14860            state.draining_to_replace =
14861                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
14862            if let Some(enabled) = enabled {
14863                state.enabled = enabled;
14864            }
14865        })?;
14866    }
14867
14868    let Some(target) = drain_target.as_ref() else {
14869        // Nothing was sent: the module has no registered connection to carry
14870        // `module.draining` or a GOODBYE. The caller must not assume the child
14871        // was asked to stop.
14872        return Ok(StopNotice::NoConnection);
14873    };
14874    let flush_goodbye = runtime.supervisor_handle.as_ref().is_some_and(|handle| {
14875        handle
14876            .registration_spawn(target.endpoint.connection_id)
14877            .is_some()
14878    });
14879    {
14880        send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
14881        #[cfg(feature = "test-support")]
14882        forwarding.wait_before_route_closing_for_test().await;
14883        let routes = forwarding
14884            .endpoint_routes(target.endpoint)
14885            .map_err(SuperviseError::Forwarding)?;
14886        let routes_notified = routes.len();
14887        crate::control::send_route_control_pushes(
14888            forwarding,
14889            routes.clone(),
14890            ClientControlPush::RouteClosing {
14891                module_id: spec.module_id.clone(),
14892                channels: Vec::new(),
14893                reason,
14894            },
14895        );
14896        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
14897
14898        // `route.closing` was just sent above: from here on every return path,
14899        // including an early one, MUST send `route.closed` before propagating
14900        // anything else. A client holds `closing` as a promise that a verdict is
14901        // coming; leaving early without `closed` strands it waiting forever, since
14902        // `closing` carries no timeout of its own.
14903        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
14904        let wait_result = wait_for_forwarding_quiescence(
14905            forwarding,
14906            &spec.module_id,
14907            runtime,
14908            target.endpoint,
14909            drain_deadline,
14910            &busy_gauges,
14911            scope,
14912            &mut gauge_trace,
14913        )
14914        .await;
14915        let drained = drained_after_quiescence_wait(&wait_result);
14916        if let Err(err) = &wait_result {
14917            error!(
14918                module_id = %spec.module_id,
14919                ?reason,
14920                error = %err,
14921                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
14922            );
14923        } else if !drained {
14924            // Read before teardown releases the routes.
14925            let holdouts = forwarding
14926                .endpoint_drain_holdouts(target.endpoint)
14927                .unwrap_or_default();
14928            warn_drain_timed_out(
14929                &spec.module_id,
14930                drain_timeout,
14931                reason,
14932                &holdouts,
14933                &gauge_trace,
14934            );
14935        }
14936        crate::control::send_route_control_pushes(
14937            forwarding,
14938            routes,
14939            ClientControlPush::RouteClosed {
14940                module_id: spec.module_id.clone(),
14941                channels: Vec::new(),
14942                reason,
14943                drained,
14944                abandoned: target.abandoned_bindings.len() as u32,
14945                excluded_subscriptions: target.excluded_subscriptions,
14946                terminal: Some(terminal),
14947            },
14948        );
14949        wait_result?;
14950
14951        // `route.closed` has now been sent unconditionally above. From here the
14952        // remaining steps are cleanup (route + module GOODBYE) rather than a
14953        // promise the client is waiting on, but a lock-poisoned
14954        // `release_module_endpoint_routes` would otherwise skip the module
14955        // GOODBYE silently too -- send it before propagating the error.
14956        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
14957            Ok(routes) => routes,
14958            Err(err) => {
14959                warn!(
14960                    module_id = %spec.module_id,
14961                    ?reason,
14962                    error = %err,
14963                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
14964                );
14965                send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
14966                return Err(SuperviseError::Forwarding(err));
14967            }
14968        };
14969        let route_goodbye_count = released_routes.len();
14970        send_route_goodbyes(forwarding, released_routes);
14971        send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
14972
14973        // The drain's happy path was previously silent: every emission above is
14974        // best-effort with only its failure arm logged, so "were consumers told"
14975        // was unprovable from the daemon log (surfaced by a 30-minute consumer
14976        // hang where the open question was exactly whether teardown notice went
14977        // out). One summary line makes that class decidable in one grep.
14978        info!(
14979            module_id = %spec.module_id,
14980            ?reason,
14981            routes_notified,
14982            route_goodbyes = route_goodbye_count,
14983            abandoned_reservations = target.abandoned_bindings.len(),
14984            excluded_subscriptions = target.excluded_subscriptions,
14985            drained,
14986            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
14987        );
14988    }
14989
14990    Ok(StopNotice::SentOverConnection)
14991}
14992
14993/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
14994/// the only slot a plain (non-swap) spawn can register into.
14995async fn wait_for_registration_after_reload(
14996    registry: &Registry,
14997    module_id: &str,
14998    snapshot: &SharedSnapshot,
14999    child: &mut SupervisedChild,
15000    wait: Duration,
15001) -> Result<RegistrationWaitOutcome, SuperviseError> {
15002    wait_for_slot_registration(
15003        registry,
15004        crate::registry::RegistrationSlot::Active(module_id),
15005        module_id,
15006        snapshot,
15007        child,
15008        wait,
15009    )
15010    .await
15011}
15012
15013/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
15014///
15015/// Keyed on the slot rather than the bare module id because during a swap the
15016/// id's active slot is already held by the incumbent: an id-keyed wait would
15017/// report the incumbent's registration as the candidate's and a candidate that
15018/// never registers would look registered. A swap candidate waits on
15019/// `crate::registry::RegistrationSlot::Candidate`.
15020async fn wait_for_slot_registration(
15021    registry: &Registry,
15022    slot: crate::registry::RegistrationSlot<'_>,
15023    module_id: &str,
15024    snapshot: &SharedSnapshot,
15025    child: &mut SupervisedChild,
15026    wait: Duration,
15027) -> Result<RegistrationWaitOutcome, SuperviseError> {
15028    let deadline = Instant::now() + wait;
15029    loop {
15030        if registry
15031            .registration(slot)
15032            .map_err(SuperviseError::Registry)?
15033            .is_some()
15034        {
15035            return Ok(RegistrationWaitOutcome::Registered);
15036        }
15037
15038        let now = Instant::now();
15039        if now >= deadline {
15040            return Ok(RegistrationWaitOutcome::TimedOut);
15041        }
15042        let remaining = deadline.saturating_duration_since(now);
15043        let poll = remaining.min(REGISTRY_RELEASE_POLL);
15044
15045        tokio::select! {
15046            wait_result = child.wait() => {
15047                let status = wait_result.map_err(|source| SuperviseError::Wait {
15048                    module_id: module_id.to_string(),
15049                    source,
15050                })?;
15051                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
15052                    snapshot,
15053                    child,
15054                    &status,
15055                )));
15056            }
15057            _ = sleep(poll) => {}
15058        }
15059    }
15060}
15061
15062fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
15063    // A process stopped for missing its first HELLO did not provide service,
15064    // even if its TERM handler exits zero. Reload also uses this classification
15065    // when a replacement exits before registering.
15066    if exit_report.kind != ExitKind::DeliberateSeverance {
15067        exit_report.kind = ExitKind::Crash;
15068    }
15069    exit_report
15070}
15071
15072async fn handle_reload_child_registration_failure(
15073    spec: &ModuleSpec,
15074    runtime: &SupervisorRuntimeConfig,
15075    registry: &Registry,
15076    process_liveness: &SupervisorProcessLiveness,
15077    snapshot: &SharedSnapshot,
15078    _child: &mut Option<SupervisedChild>,
15079    failure: ReloadRegistrationFailure,
15080) -> Result<(), SuperviseError> {
15081    let ReloadRegistrationFailure {
15082        exit_report,
15083        reason,
15084    } = failure;
15085    match on_child_exit(
15086        spec,
15087        runtime.restart_policy,
15088        registry,
15089        snapshot,
15090        &runtime.terminal_ring,
15091        &runtime.spawn_events,
15092        &runtime.child_roster,
15093        exit_report,
15094        runtime.recovery_is_daemon,
15095    )
15096    .await
15097    {
15098        NextAction::Stop {
15099            registration_released,
15100        } => {
15101            if registration_released {
15102                process_liveness.untrack_if_current(&spec.module_id, snapshot);
15103            }
15104        }
15105        NextAction::Restart { schedule } => {
15106            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
15107                schedule.delay
15108            });
15109            if let Some(schedule) = schedule {
15110                log_crash_respawn(&spec.module_id, schedule);
15111            }
15112            schedule_respawn(
15113                runtime,
15114                snapshot,
15115                &spec.module_id,
15116                delay,
15117                RespawnKind::Spawn,
15118                true,
15119            )?;
15120        }
15121    }
15122    Err(SuperviseError::ReloadFailed {
15123        module_id: spec.module_id.clone(),
15124        reason,
15125    })
15126}
15127
15128async fn handle_reload_spawn_failure(
15129    spec: &ModuleSpec,
15130    runtime: &SupervisorRuntimeConfig,
15131    process_liveness: &SupervisorProcessLiveness,
15132    snapshot: &SharedSnapshot,
15133    _child: &mut Option<SupervisedChild>,
15134    reason: String,
15135) -> Result<(), SuperviseError> {
15136    let now = Instant::now();
15137    let mut schedule = None;
15138    update_snapshot(snapshot, Some(&spec.module_id), |state| {
15139        clear_current_process_facts(state);
15140        if state.enabled {
15141            schedule = state.next_crash_restart(&runtime.restart_policy, now);
15142            state.state = if schedule.is_some() {
15143                ModuleState::Restarting
15144            } else {
15145                if runtime
15146                    .failure_context(runtime.recovery_is_daemon)
15147                    .may_retry()
15148                {
15149                    state.failed_retry.schedule(&spec.module_id);
15150                }
15151                ModuleState::Failed
15152            };
15153        } else {
15154            state.state = ModuleState::Disabled;
15155        }
15156    })?;
15157    if let Some(schedule) = schedule {
15158        schedule_respawn(
15159            runtime,
15160            snapshot,
15161            &spec.module_id,
15162            schedule.delay,
15163            RespawnKind::Spawn,
15164            true,
15165        )?;
15166    } else {
15167        process_liveness.untrack_if_current(&spec.module_id, snapshot);
15168    }
15169    Err(SuperviseError::ReloadFailed {
15170        module_id: spec.module_id.clone(),
15171        reason,
15172    })
15173}
15174
15175fn control_flags() -> Flags {
15176    Flags::new(false, Priority::Passive, false)
15177}
15178
15179#[allow(clippy::too_many_arguments)]
15180async fn drain_optional_child(
15181    module_id: &str,
15182    protocol: ModuleProtocol,
15183    stop_notice: StopNotice,
15184    registry: &Registry,
15185    forwarding: Option<&ForwardingTable>,
15186    snapshot: &SharedSnapshot,
15187    terminal_ring: &Arc<Mutex<TerminalRing>>,
15188    spawn_events: &SpawnEventFeed,
15189    child: &mut Option<SupervisedChild>,
15190    drain_timeout: Duration,
15191    final_state: ModuleState,
15192    enabled: Option<bool>,
15193    failure: &FailureContext,
15194) -> Result<(), SuperviseError> {
15195    if let Some(child) = child.take() {
15196        drain_child_to_state(
15197            module_id,
15198            protocol,
15199            stop_notice,
15200            registry,
15201            forwarding,
15202            snapshot,
15203            terminal_ring,
15204            spawn_events,
15205            child,
15206            drain_timeout,
15207            final_state,
15208            enabled,
15209            failure,
15210        )
15211        .await
15212    } else {
15213        update_snapshot(snapshot, Some(module_id), |state| {
15214            state.state = final_state;
15215            if final_state == ModuleState::Failed {
15216                state.failed_retry.failed_at = Some(Instant::now());
15217            }
15218            if let Some(enabled) = enabled {
15219                state.enabled = enabled;
15220            }
15221            clear_current_process_facts(state);
15222        })?;
15223        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
15224    }
15225}
15226
15227#[allow(clippy::too_many_arguments)]
15228async fn drain_child_to_state(
15229    module_id: &str,
15230    _protocol: ModuleProtocol,
15231    stop_notice: StopNotice,
15232    registry: &Registry,
15233    forwarding: Option<&ForwardingTable>,
15234    snapshot: &SharedSnapshot,
15235    terminal_ring: &Arc<Mutex<TerminalRing>>,
15236    spawn_events: &SpawnEventFeed,
15237    mut child: SupervisedChild,
15238    drain_timeout: Duration,
15239    final_state: ModuleState,
15240    enabled: Option<bool>,
15241    failure: &FailureContext,
15242) -> Result<(), SuperviseError> {
15243    let protocol = child.protocol;
15244    child.retire_registration();
15245    update_snapshot(snapshot, Some(module_id), |state| {
15246        state.state = ModuleState::Draining;
15247        state.draining_to_replace = final_state == ModuleState::Restarting;
15248        if let Some(enabled) = enabled {
15249            state.enabled = enabled;
15250        }
15251    })?;
15252
15253    // The wait below is the same budget in every case; what differs is
15254    // whether anything has ASKED the child to stop before it starts. Only a
15255    // forwarding drain that reached the module's registered connection has
15256    // (`module.draining`, then a module GOODBYE). Every other child was told
15257    // nothing: a `protocol: "none"` module, which never registers; a subc
15258    // module spawned moments ago that has not sent HELLO yet; or a stop that
15259    // runs no forwarding drain. Without a signal the budget is only a delay
15260    // in front of SIGKILL -- and the not-yet-registered child is the worst
15261    // case, because it registers into a module that is already draining,
15262    // is never told, and is killed while healthy.
15263    if stop_notice != StopNotice::SentOverConnection {
15264        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
15265            info!(
15266                module_id,
15267                pid = child.pid,
15268                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15269                "module has no connection yet; requesting stop by signal"
15270            );
15271        }
15272        request_graceful_stop(module_id, &child);
15273    }
15274
15275    let exit_report = match timeout(drain_timeout, child.wait()).await {
15276        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
15277        Ok(Err(source)) => {
15278            fail_after_wait_error(
15279                module_id,
15280                snapshot,
15281                terminal_ring,
15282                spawn_events,
15283                format!("failed to wait during drain: {source}"),
15284                failure,
15285            );
15286            return Err(SuperviseError::Wait {
15287                module_id: module_id.to_string(),
15288                source,
15289            });
15290        }
15291        Err(_) => {
15292            // Mirror the sibling arm above: state is already `Draining`, and an
15293            // error propagated from here would strand it there -- a state
15294            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
15295            // `Failed | Stopped`), leaving an operator Restart as the only exit.
15296            // `Failed` before `?` keeps the module operator-visible and
15297            // revivable. Trigger is an ESRCH race (process exits between the
15298            // drain timeout firing and the kill) or a post-kill wait failure
15299            // (issue #34).
15300            //
15301            // Logged because the kill is otherwise visible only as signal 9 in
15302            // the terminal ring, and the budget it follows can be long enough
15303            // that consumers see a stretch of refusals with no stated cause.
15304            warn!(
15305                module_id,
15306                pid = child.pid,
15307                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15308                reason = ?final_state,
15309                ?stop_notice,
15310                "drain budget expired before the module exited; killing it"
15311            );
15312            let kill =
15313                if final_state == ModuleState::Restarting && child.child.container().is_some() {
15314                    child.kill_leader()
15315                } else {
15316                    child.start_kill()
15317                };
15318            kill.map_err(|source| {
15319                fail_after_wait_error(
15320                    module_id,
15321                    snapshot,
15322                    terminal_ring,
15323                    spawn_events,
15324                    format!("failed to kill during drain: {source}"),
15325                    failure,
15326                );
15327                SuperviseError::Kill {
15328                    module_id: module_id.to_string(),
15329                    source,
15330                }
15331            })?;
15332            let status = child.wait().await.map_err(|source| {
15333                fail_after_wait_error(
15334                    module_id,
15335                    snapshot,
15336                    terminal_ring,
15337                    spawn_events,
15338                    format!("failed to wait after drain kill: {source}"),
15339                    failure,
15340                );
15341                SuperviseError::Wait {
15342                    module_id: module_id.to_string(),
15343                    source,
15344                }
15345            })?;
15346            classify_reaped_child_exit(snapshot, &child, &status)
15347        }
15348    };
15349
15350    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
15351    publish_after_terminal_record(
15352        module_id,
15353        || {
15354            record_terminal_with_detail(
15355                module_id,
15356                terminal_ring,
15357                spawn_events,
15358                &exit_report,
15359                terminal_disposition(final_state),
15360                detail,
15361            )
15362        },
15363        || {
15364            update_snapshot(snapshot, Some(module_id), |state| {
15365                state.state = final_state;
15366                if final_state == ModuleState::Failed {
15367                    state.failed_retry.failed_at = Some(Instant::now());
15368                }
15369                if let Some(enabled) = enabled {
15370                    state.enabled = enabled;
15371                }
15372                clear_current_process_facts(state);
15373                state.last_exit = Some(exit_report.clone());
15374                if exit_report.kind == ExitKind::DeliberateSeverance {
15375                    state.lifetime_restarts += 1;
15376                }
15377            })
15378        },
15379    )?;
15380    child.drain_stderr(module_id).await;
15381
15382    let released =
15383        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
15384    child
15385        .finish_exit(
15386            released.is_ok()
15387                && final_state == ModuleState::Restarting
15388                && child.spawn_failure.is_none(),
15389        )
15390        .await
15391        .map_err(|source| SuperviseError::Wait {
15392            module_id: module_id.to_owned(),
15393            source,
15394        })?;
15395    released
15396}
15397
15398/// Ask a child that nothing else has asked to stop, by signal.
15399///
15400/// A registered subc module is asked over its own connection: the drain sends
15401/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
15402/// module GOODBYE, and the module stops itself. A module that speaks no subc
15403/// wire receives none of that, and neither does a subc module that has not
15404/// registered yet, so for them the drain budget would be pure delay in front of
15405/// a SIGKILL -- and for a process with a store to flush (JetStream is the
15406/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
15407/// into a recovery on the next start.
15408///
15409/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
15410/// rule rather than an optimisation: that module's graceful stop is already
15411/// running by the time its child is drained, and a signal would race it.
15412///
15413/// Best-effort by construction. A child that has already exited is the ordinary
15414/// case rather than an error (the kill lands on a reaped or exiting pid), so a
15415/// failure is logged at debug and the wait-then-kill below still decides the
15416/// outcome.
15417#[cfg(unix)]
15418fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
15419    let Some(pid) = child
15420        .id()
15421        .and_then(|pid| i32::try_from(pid).ok())
15422        .and_then(rustix::process::Pid::from_raw)
15423    else {
15424        debug!(
15425            module_id,
15426            "no pid to signal for teardown; falling through to the drain wait"
15427        );
15428        return;
15429    };
15430    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
15431        Ok(()) => debug!(
15432            module_id,
15433            "sent SIGTERM to a module nothing else asked to stop"
15434        ),
15435        Err(err) => debug!(
15436            module_id,
15437            error = %err,
15438            "SIGTERM to module failed; the drain wait and kill still apply"
15439        ),
15440    }
15441}
15442
15443/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
15444/// Windows does offer need cooperation this supervisor cannot assume: a console
15445/// control event requires sharing a console with the child, and `WM_CLOSE`
15446/// requires the child to pump a message loop. A supervised server process does
15447/// neither, so there is nothing to send and teardown is the wait followed by the
15448/// kill. Emulating a signal here would mean inventing a stop protocol, which is
15449/// the thing `protocol: "none"` exists to avoid.
15450#[cfg(not(unix))]
15451fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
15452    debug!(
15453        module_id,
15454        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
15455    );
15456}
15457
15458fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
15459    match final_state {
15460        ModuleState::Stopped => TerminalDisposition::Stopped,
15461        ModuleState::Disabled => TerminalDisposition::Disabled,
15462        ModuleState::Restarting => TerminalDisposition::Restarting,
15463        ModuleState::Failed => TerminalDisposition::Failed,
15464        ModuleState::Starting
15465        | ModuleState::Running
15466        | ModuleState::Unresponsive
15467        | ModuleState::Draining => {
15468            unreachable!("terminal exits only finish in terminal or restarting states")
15469        }
15470    }
15471}
15472
15473/// Release a reaped child's registration before allowing another spawn.
15474///
15475/// EOF is not a process-lifetime signal: an inherited socket can stay open
15476/// indefinitely, and serial frame dispatch can be waiting on egress instead of
15477/// reading EOF. After the normal release grace, request connection close (which
15478/// cancels both reads and dispatch), then allow one more release grace for the
15479/// connection guard's forwarding cleanup. Never evict a different connection.
15480async fn release_dead_registration(
15481    registry: &Registry,
15482    forwarding: Option<&ForwardingTable>,
15483    snapshot: &SharedSnapshot,
15484    module_id: &str,
15485    failure: &FailureContext,
15486) -> Result<(), SuperviseError> {
15487    let result = async {
15488        let registration = registry
15489            .get_module(module_id)
15490            .map_err(SuperviseError::Registry)?;
15491        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
15492            Ok(()) => return Ok(()),
15493            Err(SuperviseError::RegistrationStillActive { .. }) => {}
15494            Err(err) => return Err(err),
15495        }
15496        let pid = lock_snapshot(snapshot)?.reaped_pid;
15497        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
15498            warn!(
15499                module_id,
15500                pid,
15501                connection_id = registration.connection_id.get(),
15502                "reaped module registration outlived release grace; closing dead connection"
15503            );
15504            forwarding.request_connection_close(
15505                registration.connection_id,
15506                CloseReason::new(
15507                    "supervised_process_reaped",
15508                    format!("module '{module_id}' pid {pid} exited"),
15509                ),
15510            );
15511            wait_for_slot_registration_release(
15512                registry,
15513                crate::registry::RegistrationSlot::Connection(registration.connection_id),
15514                REGISTRY_RELEASE_TIMEOUT,
15515            )
15516            .await?;
15517        }
15518        wait_for_registration_release(registry, module_id, Duration::ZERO).await
15519    }
15520    .await;
15521    if let Err(err) = &result {
15522        fail_snapshot(snapshot, Some(module_id), None, failure);
15523        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
15524    }
15525    result
15526}
15527
15528/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
15529/// plain stop or restart waits for before it spawns a replacement.
15530async fn wait_for_registration_release(
15531    registry: &Registry,
15532    module_id: &str,
15533    wait: Duration,
15534) -> Result<(), SuperviseError> {
15535    wait_for_slot_registration_release(
15536        registry,
15537        crate::registry::RegistrationSlot::Active(module_id),
15538        wait,
15539    )
15540    .await
15541}
15542
15543/// Wait for the registration in `slot` to go away.
15544///
15545/// Keyed on the slot rather than the bare module id because a successful swap
15546/// never empties the id's active slot (the promoted candidate is in it), so an
15547/// id-keyed wait for the incumbent's release would always time out. Draining a
15548/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
15549/// incumbent's connection instead.
15550async fn wait_for_slot_registration_release(
15551    registry: &Registry,
15552    slot: crate::registry::RegistrationSlot<'_>,
15553    wait: Duration,
15554) -> Result<(), SuperviseError> {
15555    let deadline = Instant::now() + wait;
15556    let mut release_events = registration_release_events().subscribe();
15557    let still_active = |registration: &crate::registry::ModuleRegistration| {
15558        SuperviseError::RegistrationStillActive {
15559            module_id: registration.manifest.module_id.clone(),
15560            waited: wait,
15561        }
15562    };
15563    loop {
15564        let _observed_generation = *release_events.borrow_and_update();
15565        let Some(registration) = registry
15566            .registration(slot)
15567            .map_err(SuperviseError::Registry)?
15568        else {
15569            return Ok(());
15570        };
15571
15572        let now = Instant::now();
15573        if now >= deadline {
15574            return Err(still_active(&registration));
15575        }
15576
15577        let remaining = deadline.saturating_duration_since(now);
15578        match timeout(remaining, release_events.changed()).await {
15579            Ok(Ok(())) | Ok(Err(_)) => {}
15580            Err(_) => return Err(still_active(&registration)),
15581        }
15582    }
15583}
15584
15585#[cfg(test)]
15586mod slot_registration_wait_tests {
15587    use super::*;
15588    use crate::registry::{ConnectionId, RegistrationSlot};
15589    use subc_protocol::manifest::ModuleManifest;
15590
15591    #[tokio::test]
15592    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
15593        let registry = Arc::new(Registry::default());
15594        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
15595        let runtime = supervisor.runtime_config();
15596        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
15597        let spec = ModuleSpec {
15598            script: None,
15599            module_id: "enable-stale-registration".to_string(),
15600            program: PathBuf::from("/missing/enable-retry-test"),
15601            args: Vec::new(),
15602            env: Vec::new(),
15603            reserved: false,
15604            reserved_prefixes: Vec::new(),
15605            protocol: ModuleProtocol::Subc,
15606            overlap: Default::default(),
15607            carry_containment: false,
15608        };
15609        let connection = ConnectionId::new(90);
15610        registry
15611            .register_with_control_ops(
15612                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
15613                1,
15614                connection,
15615                Vec::new(),
15616            )
15617            .unwrap();
15618        let mut child = None;
15619        let err = set_child_enabled(
15620            &spec,
15621            &runtime,
15622            &registry,
15623            &supervisor.process_liveness,
15624            &snapshot,
15625            &mut child,
15626            true,
15627            &runtime.failure_context(false),
15628        )
15629        .await
15630        .unwrap_err();
15631        assert!(matches!(
15632            err,
15633            SuperviseError::RegistrationStillActive { .. }
15634        ));
15635        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15636        assert!(child.is_none());
15637        registry.deregister_connection(connection).unwrap();
15638        let err = set_child_enabled(
15639            &spec,
15640            &runtime,
15641            &registry,
15642            &supervisor.process_liveness,
15643            &snapshot,
15644            &mut child,
15645            true,
15646            &runtime.failure_context(false),
15647        )
15648        .await
15649        .unwrap_err();
15650        assert!(
15651            matches!(err, SuperviseError::Spawn { .. }),
15652            "second enable must attempt a spawn: {err}"
15653        );
15654        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15655    }
15656
15657    const INCUMBENT: u64 = 1;
15658    const CANDIDATE: u64 = 2;
15659
15660    fn swapped_registry() -> Arc<Registry> {
15661        let registry = Arc::new(Registry::default());
15662        let manifest = ModuleManifest::builder("m", "0.1.0").build();
15663        registry
15664            .register_with_control_ops(
15665                manifest.clone(),
15666                1,
15667                ConnectionId::new(INCUMBENT),
15668                Vec::new(),
15669            )
15670            .unwrap();
15671        registry
15672            .register_candidate_with_control_ops(
15673                manifest,
15674                1,
15675                ConnectionId::new(CANDIDATE),
15676                Vec::new(),
15677            )
15678            .unwrap();
15679        registry
15680    }
15681
15682    /// After a promotion the id's active slot is held by the new process, so an
15683    /// id-keyed wait for the incumbent's release can never succeed; the
15684    /// connection-keyed wait completes as soon as the incumbent deregisters.
15685    #[tokio::test]
15686    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
15687        let registry = swapped_registry();
15688        registry.promote_candidate("m").unwrap().unwrap();
15689
15690        assert!(matches!(
15691            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
15692            Err(SuperviseError::RegistrationStillActive { .. })
15693        ));
15694
15695        // Still held while the incumbent's connection has not deregistered.
15696        assert!(matches!(
15697            wait_for_slot_registration_release(
15698                &registry,
15699                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15700                Duration::from_millis(50),
15701            )
15702            .await,
15703            Err(SuperviseError::RegistrationStillActive { .. })
15704        ));
15705
15706        let releaser = Arc::clone(&registry);
15707        let release = tokio::spawn(async move {
15708            sleep(Duration::from_millis(20)).await;
15709            releaser
15710                .deregister_connection(ConnectionId::new(INCUMBENT))
15711                .unwrap();
15712            notify_registration_release();
15713        });
15714        wait_for_slot_registration_release(
15715            &registry,
15716            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15717            Duration::from_secs(5),
15718        )
15719        .await
15720        .expect("the incumbent's own registration is released");
15721        release.await.unwrap();
15722        assert!(registry.get_module("m").unwrap().is_some());
15723    }
15724
15725    /// The candidate slot is waited on separately from the active slot: the
15726    /// incumbent's registration neither holds up nor stands in for it.
15727    #[tokio::test]
15728    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
15729        let registry = swapped_registry();
15730        assert!(matches!(
15731            wait_for_slot_registration_release(
15732                &registry,
15733                RegistrationSlot::Candidate("m"),
15734                Duration::from_millis(50),
15735            )
15736            .await,
15737            Err(SuperviseError::RegistrationStillActive { .. })
15738        ));
15739        registry
15740            .deregister_connection(ConnectionId::new(CANDIDATE))
15741            .unwrap();
15742        wait_for_slot_registration_release(
15743            &registry,
15744            RegistrationSlot::Candidate("m"),
15745            Duration::from_millis(50),
15746        )
15747        .await
15748        .expect("a candidate slot with no candidate is released");
15749        assert!(registry
15750            .registration(RegistrationSlot::Active("m"))
15751            .unwrap()
15752            .is_some());
15753    }
15754}
15755
15756fn classify_exit(status: &ExitStatus) -> ExitReport {
15757    ExitReport {
15758        kind: if status.success() {
15759            ExitKind::Clean
15760        } else {
15761            ExitKind::Crash
15762        },
15763        code: status.code(),
15764        signal: exit_signal(status),
15765        at_ms: unix_ms_now(),
15766    }
15767}
15768
15769/// The terminal record for a module whose `wait()` call itself errored (e.g. the
15770/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
15771/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
15772/// disposition still must be `Failed` so the terminal ring is not silently missing
15773/// an entry, matching what `fail_snapshot` records for this same arm.
15774fn wait_error_exit_report() -> ExitReport {
15775    ExitReport {
15776        kind: ExitKind::Crash,
15777        code: None,
15778        signal: None,
15779        at_ms: unix_ms_now(),
15780    }
15781}
15782
15783#[cfg(unix)]
15784fn exit_signal(status: &ExitStatus) -> Option<i32> {
15785    use std::os::unix::process::ExitStatusExt;
15786
15787    status.signal()
15788}
15789
15790#[cfg(not(unix))]
15791fn exit_signal(_status: &ExitStatus) -> Option<i32> {
15792    None
15793}
15794
15795/// Give an operator-touched module its full crash budget back.
15796///
15797/// Named for the counter it used to zero; it now empties the in-window ring,
15798/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
15799/// ledger of what happened survives every operator action.
15800fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
15801    update_snapshot(snapshot, Some(module_id), |state| {
15802        state.clear_crash_restarts();
15803    })
15804}
15805
15806fn set_running(
15807    snapshot: &SharedSnapshot,
15808    child: &SupervisedChild,
15809    module_id: &str,
15810    spawn_events: &SpawnEventFeed,
15811) -> Result<(), SuperviseError> {
15812    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
15813        module_id: Some(module_id.to_string()),
15814    })?;
15815    state.spawn_generation = spawn_events.emit_spawned(
15816        module_id,
15817        child
15818            .registration_spawn
15819            .as_ref()
15820            .map(|spawn| spawn.authority.spawn_number()),
15821        child.pid,
15822        child.spawned_at_ms,
15823    );
15824    if std::mem::take(&mut state.coalesced_restart_pending) {
15825        let generation = state.spawn_generation;
15826        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
15827    }
15828    state.drain_disposition_detail = None;
15829    state.spawn_failure = None;
15830    state.admission_refused = None;
15831    state.config_hold_parked = false;
15832    state.config_hold_daemon_initiated = false;
15833    state.config_hold_approval = None;
15834    state.legacy_registration_spawn =
15835        child.registration_spawn.is_some() && child.admission_base.is_none();
15836    state.registration_declaration = child.registration_declaration.clone();
15837    state.stop_reason = None;
15838    state.config_exit_failed_approval = None;
15839    state.waiting_for_previous_pid = None;
15840    state.waiting_for_previous_detail = None;
15841    // Every caller of this is a plain spawn, which always uses the primary key;
15842    // a promoted swap candidate sets the flag itself after this returns.
15843    state.in_alternate_slot = false;
15844    state.configuration_updated_since_spawn = false;
15845    state.spawned_protocol = Some(child.protocol);
15846    state.state = ModuleState::Running;
15847    state.approved_start_failure = None;
15848    state.enabled = true;
15849    state.process_alive = true;
15850    state.pid = child.id();
15851    state.subtree_target = child.subtree_target.clone();
15852    // Probe samples belong to one process. Keep module-level escalation and
15853    // late-answer history, but never charge a new child for its predecessor's misses.
15854    state.health.status = SupervisorHealthStatus::Unknown;
15855    state.last_probe_evidence = None;
15856    state.health.consecutive_failures = 0;
15857    state.health.detail = None;
15858    state.health.metrics = None;
15859    state.health.last_probe_ms = None;
15860    state.failed_retry.healthy_since = None;
15861    state.failed_retry.failed_at = None;
15862    state.failed_retry.running_since = Some(child.spawned_at);
15863    #[cfg(windows)]
15864    {
15865        state.pipe_consumed = child
15866            .nonce_delivery
15867            .as_ref()
15868            .map(|delivery| delivery.consumed());
15869    }
15870    #[cfg(target_os = "macos")]
15871    {
15872        state.report_ready = Some(Arc::clone(&child.report_ready));
15873    }
15874    state.spawned_at_ms = Some(child.spawned_at_ms);
15875    state.exec_outcome = Some(Arc::clone(&child.exec.0));
15876    state.spawned_from = Some(child.spawned_from.clone());
15877    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
15878    state.spawned_file_identity = child.spawned_file_identity.clone();
15879    state.process_start_time = child.process_start_time;
15880    Ok(())
15881}
15882
15883fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
15884    state.legacy_registration_spawn = false;
15885    state.registration_declaration = None;
15886    state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
15887    state.failed_retry.healthy_since = None;
15888    state.failed_retry.running_since = None;
15889    state.waiting_for_previous_pid = None;
15890    state.waiting_for_previous_detail = None;
15891    state.process_alive = false;
15892    state.spawned_protocol = None;
15893    state.pid = None;
15894    #[cfg(windows)]
15895    {
15896        state.pipe_consumed = None;
15897    }
15898    #[cfg(target_os = "macos")]
15899    {
15900        state.report_ready = None;
15901    }
15902    state.spawned_at_ms = None;
15903    state.exec_outcome = None;
15904    state.spawned_from = None;
15905    state.launch_spec_digest = None;
15906    state.spawned_file_identity = None;
15907    state.process_start_time = None;
15908    state.deliberate_severance = None;
15909}
15910
15911#[cfg(test)]
15912fn record_deliberate_severance(
15913    snapshot: &SharedSnapshot,
15914    identity: ProcessIdentity,
15915) -> Result<(), SuperviseError> {
15916    update_snapshot(snapshot, None, |state| {
15917        state.deliberate_severance = Some(identity);
15918    })
15919}
15920
15921fn apply_deliberate_severance_marker(
15922    snapshot: &SharedSnapshot,
15923    exited_identity: Option<ProcessIdentity>,
15924    mut exit_report: ExitReport,
15925) -> ExitReport {
15926    let marker = lock_snapshot(snapshot)
15927        .ok()
15928        .and_then(|mut state| state.deliberate_severance.take());
15929    if marker.is_some() && marker == exited_identity {
15930        exit_report.kind = ExitKind::DeliberateSeverance;
15931    }
15932    exit_report
15933}
15934
15935fn classify_reaped_child_exit(
15936    snapshot: &SharedSnapshot,
15937    child: &SupervisedChild,
15938    status: &ExitStatus,
15939) -> ExitReport {
15940    let _ = update_snapshot(snapshot, None, |state| {
15941        state.reaped_pid = Some(child.pid);
15942        state.spawn_failure = child.spawn_failure.clone();
15943    });
15944    if matches!(
15945        child.spawn_failure.as_deref(),
15946        Some("registration_timeout" | "registration_channel_closed")
15947    ) {
15948        registration_failure_exit_report(classify_exit(status))
15949    } else {
15950        apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
15951    }
15952}
15953
15954fn fail_snapshot(
15955    snapshot: &SharedSnapshot,
15956    module_id: Option<&str>,
15957    last_exit: Option<ExitReport>,
15958    failure: &FailureContext,
15959) {
15960    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
15961        state.state = ModuleState::Failed;
15962        state.config_hold_parked = false;
15963        state.config_hold_daemon_initiated = false;
15964        state.config_hold_approval = None;
15965        clear_current_process_facts(state);
15966        if failure.may_retry() && state.enabled {
15967            state.failed_retry.schedule(module_id.unwrap_or("unknown"));
15968        } else if failure.shutdown.is_set() {
15969            state.failed_retry.cancel();
15970        }
15971        if let Some(last_exit) = last_exit {
15972            state.last_exit = Some(last_exit);
15973        }
15974    }) {
15975        error!(error = %err, "failed to mark supervisor state failed");
15976    }
15977}
15978
15979fn update_snapshot(
15980    snapshot: &SharedSnapshot,
15981    module_id: Option<&str>,
15982    update: impl FnOnce(&mut SupervisorSnapshot),
15983) -> Result<(), SuperviseError> {
15984    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
15985        module_id: module_id.map(ToOwned::to_owned),
15986    })?;
15987    update(&mut state);
15988    Ok(())
15989}
15990
15991const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
15992
15993fn lock_snapshot_for_control<'a>(
15994    snapshot: &'a SharedSnapshot,
15995    module_id: &str,
15996    caller: &'static str,
15997) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
15998    let started_at = Instant::now();
15999    let guard = lock_snapshot(snapshot)?;
16000    let waited = started_at.elapsed();
16001    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
16002        warn!(
16003            module_id = %module_id,
16004            waited_ms = waited.as_millis() as u64,
16005            caller = %caller,
16006            "slow snapshot lock"
16007        );
16008    }
16009    Ok(guard)
16010}
16011
16012fn lock_snapshot(
16013    snapshot: &SharedSnapshot,
16014) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
16015    snapshot
16016        .lock()
16017        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
16018}
16019
16020#[cfg(test)]
16021mod terminal_history_tests {
16022    include!("supervise_route_open_recovery_tests.rs");
16023    use std::{
16024        path::PathBuf,
16025        sync::Arc,
16026        time::{Duration, Instant},
16027    };
16028
16029    use tokio::{sync::mpsc, time::sleep};
16030
16031    use super::{
16032        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
16033        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
16034        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
16035        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
16036        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
16037        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
16038        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
16039        SupervisorHealthStatus, SupervisorSnapshot,
16040    };
16041    // The supervisor's clock, distinct from the `std::time::Instant` these tests
16042    // use for their own wall-clock deadlines: crash-restart instants must be on
16043    // the same clock the production code stamps them with, which is tokio's (and
16044    // is what `start_paused` tests can move).
16045    use super::Instant as ClockInstant;
16046    use crate::{
16047        registry::Registry,
16048        terminal_ring::{TerminalRing, TerminalRingConfig},
16049    };
16050    use std::sync::Mutex;
16051    use subc_control::TerminalDisposition;
16052
16053    /// See the twin in `control.rs` for why this derives the path from
16054    /// `current_exe()` and why the existence check is here: `--lib` alone does
16055    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
16056    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
16057    pub(super) fn fake_aft_stub_path() -> PathBuf {
16058        let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
16059        assert!(
16060            path.exists(),
16061            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
16062             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
16063            path.display()
16064        );
16065        path
16066    }
16067
16068    #[test]
16069    fn reserved_never_spawned_refuses_every_hello() {
16070        // The canary hole: a reserved id whose module has never spawned had NO
16071        // gate entry and admitted anyone -- the reservation protected the nonce
16072        // holder, not the NAME. Now the entry is present with no legitimate
16073        // holder and refuses all comers.
16074        let supervisor = SupervisorHandle::default();
16075        supervisor.apply_identity_configuration(&ModuleSpec {
16076            script: None,
16077            module_id: "never-spawned".to_string(),
16078            program: PathBuf::from("/usr/bin/false"),
16079            args: Vec::new(),
16080            env: Vec::new(),
16081            reserved: true,
16082            reserved_prefixes: Vec::new(),
16083            protocol: ModuleProtocol::Subc,
16084            overlap: Default::default(),
16085            carry_containment: false,
16086        });
16087        assert!(
16088            supervisor
16089                .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
16090                .is_err(),
16091            "forged nonce must refuse on a reserved never-spawned id"
16092        );
16093        assert!(
16094            supervisor
16095                .reserved_hello_verification("never-spawned", None)
16096                .is_err(),
16097            "absent nonce must refuse on a reserved never-spawned id"
16098        );
16099        // And a real spawn nonce minted later admits exactly that nonce.
16100        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
16101        supervisor.apply_identity_configuration(&ModuleSpec {
16102            script: None,
16103            module_id: "never-spawned".to_string(),
16104            program: PathBuf::from("/usr/bin/false"),
16105            args: Vec::new(),
16106            env: Vec::new(),
16107            reserved: true,
16108            reserved_prefixes: Vec::new(),
16109            protocol: ModuleProtocol::Subc,
16110            overlap: Default::default(),
16111            carry_containment: false,
16112        });
16113        assert!(supervisor
16114            .reserved_hello_verification("never-spawned", Some("minted"))
16115            .is_ok());
16116        assert!(supervisor
16117            .reserved_hello_verification("never-spawned", Some("forged"))
16118            .is_err());
16119    }
16120
16121    /// Put `count` crash restarts on a snapshot's ring as if they had all just
16122    /// happened, which is what "spent budget" looks like to every reader.
16123    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
16124        let now = ClockInstant::now();
16125        for _ in 0..count {
16126            state.crash_restarts.push_back(now);
16127        }
16128    }
16129
16130    /// Age the oldest recorded restart out of `window`, standing in for the hours
16131    /// that would otherwise have to pass. Injecting the instant is the point: a
16132    /// test that slept a real window would take ten minutes and still prove less.
16133    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
16134        let aged = state
16135            .crash_restarts
16136            .front()
16137            .expect("a crash restart must be recorded before it can be aged")
16138            .checked_sub(window + Duration::from_secs(1))
16139            .expect("the test clock is far enough from its origin to age an instant");
16140        state.crash_restarts[0] = aged;
16141    }
16142
16143    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
16144        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
16145        seed_crash_restarts(&mut state, count);
16146        state
16147    }
16148
16149    #[test]
16150    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
16151        let policy = RestartPolicy::new(3, Duration::ZERO);
16152        let now = ClockInstant::now();
16153        assert!(daemon_will_restart(
16154            &mut snapshot_with_restarts(true, 2),
16155            &policy,
16156            now
16157        ));
16158        assert!(!daemon_will_restart(
16159            &mut snapshot_with_restarts(true, 3),
16160            &policy,
16161            now
16162        ));
16163        assert!(!daemon_will_restart(
16164            &mut snapshot_with_restarts(false, 0),
16165            &policy,
16166            now
16167        ));
16168    }
16169
16170    #[test]
16171    fn crash_restart_backoff_escalates_with_in_window_count() {
16172        let policy = RestartPolicy::new(4, Duration::from_millis(100))
16173            .with_max_backoff(Duration::from_secs(30));
16174        let now = ClockInstant::now();
16175        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16176        let schedules = (0..4)
16177            .map(|_| {
16178                state
16179                    .next_crash_restart(&policy, now)
16180                    .expect("the test policy allows four crash restarts")
16181            })
16182            .collect::<Vec<_>>();
16183
16184        assert_eq!(
16185            schedules
16186                .iter()
16187                .map(|schedule| schedule.restart_in_window)
16188                .collect::<Vec<_>>(),
16189            vec![0, 1, 2, 3]
16190        );
16191        assert_eq!(
16192            schedules
16193                .iter()
16194                .map(|schedule| schedule.delay)
16195                .collect::<Vec<_>>(),
16196            vec![
16197                Duration::from_millis(100),
16198                Duration::from_secs(1),
16199                Duration::from_secs(10),
16200                Duration::from_secs(30),
16201            ]
16202        );
16203    }
16204
16205    #[test]
16206    fn crash_restart_backoff_resets_after_ring_clear() {
16207        let policy = RestartPolicy::new(3, Duration::from_millis(100));
16208        let now = ClockInstant::now();
16209        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16210        assert_eq!(
16211            state.next_crash_restart(&policy, now).unwrap().delay,
16212            Duration::from_millis(100)
16213        );
16214        assert_eq!(
16215            state.next_crash_restart(&policy, now).unwrap().delay,
16216            Duration::from_secs(1)
16217        );
16218
16219        state.clear_crash_restarts();
16220        let schedule = state
16221            .next_crash_restart(&policy, now)
16222            .expect("a cleared ring must allow another restart");
16223        assert_eq!(schedule.restart_in_window, 0);
16224        assert_eq!(schedule.delay, Duration::from_millis(100));
16225    }
16226
16227    #[test]
16228    fn crash_restart_backoff_ignores_aged_restarts() {
16229        let policy = RestartPolicy::new(3, Duration::from_millis(100));
16230        let now = ClockInstant::now();
16231        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16232        state
16233            .next_crash_restart(&policy, now)
16234            .expect("the first restart is allowed");
16235        state
16236            .next_crash_restart(&policy, now)
16237            .expect("the second restart is allowed");
16238        state.crash_restarts[0] = now
16239            .checked_sub(policy.window + Duration::from_secs(1))
16240            .expect("the fake clock can age a restart past the window");
16241
16242        let schedule = state
16243            .next_crash_restart(&policy, now)
16244            .expect("an aged restart must release its slot");
16245        assert_eq!(schedule.restart_in_window, 1);
16246        assert_eq!(schedule.delay, Duration::from_secs(1));
16247        assert_eq!(state.crash_restarts.len(), 2);
16248    }
16249
16250    /// The budget is a rate: the same three spent restarts refuse a respawn
16251    /// while they are recent and allow one once they have aged past the window.
16252    /// Nothing about the module changed in between, which is the whole point.
16253    #[test]
16254    fn a_budget_spent_before_the_window_no_longer_refuses() {
16255        let policy = RestartPolicy::new(3, Duration::ZERO);
16256        let mut state = snapshot_with_restarts(true, 3);
16257        let now = ClockInstant::now();
16258        assert!(!daemon_will_restart(&mut state, &policy, now));
16259
16260        assert!(daemon_will_restart(
16261            &mut state,
16262            &policy,
16263            now + policy.window + Duration::from_secs(1)
16264        ));
16265        assert!(
16266            state.crash_restarts.is_empty(),
16267            "reading the budget must drop the instants that left the window"
16268        );
16269    }
16270
16271    fn module_with_recovery_snapshot(
16272        state: ModuleState,
16273        enabled: bool,
16274        restart_count: u32,
16275    ) -> SupervisedModule {
16276        let registry = Arc::new(Registry::default());
16277        let supervisor =
16278            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
16279        let runtime = supervisor.runtime_config();
16280        let spec = ModuleSpec {
16281            script: None,
16282            module_id: "recovery-snapshot".to_string(),
16283            program: fake_aft_stub_path(),
16284            args: Vec::new(),
16285            env: Vec::new(),
16286            reserved: false,
16287            reserved_prefixes: Vec::new(),
16288            protocol: ModuleProtocol::Subc,
16289            overlap: Default::default(),
16290            carry_containment: false,
16291        };
16292        let mut snapshot = SupervisorSnapshot::new(state, enabled);
16293        seed_crash_restarts(&mut snapshot, restart_count);
16294        // These tests read synthetic snapshots. A real child and monitor would
16295        // race those reads by replacing the requested state during startup.
16296        let (commands, _rx) = mpsc::channel(4);
16297        let configured_enabled = snapshot.enabled;
16298        SupervisedModule {
16299            inner: Arc::new(SupervisedModuleInner {
16300                module_id: spec.module_id.clone(),
16301                registry,
16302                snapshot: Arc::new(Mutex::new(snapshot)),
16303                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
16304                    spec,
16305                    enabled: configured_enabled,
16306                    health: runtime.health,
16307                    register_timeout: runtime.register_timeout,
16308                    restart_policy: runtime.restart_policy,
16309                })),
16310                stderr_ring: runtime.stderr_ring,
16311                terminal_ring: runtime.terminal_ring,
16312                commands,
16313                monitor: Mutex::new(None),
16314                effective_drain_timeout: runtime.effective_drain_timeout,
16315                provenance_probe: supervisor.provenance_probe.clone(),
16316                retained: runtime.child_roster.retained_view(),
16317            }),
16318        }
16319    }
16320
16321    #[cfg(target_os = "linux")]
16322    #[tokio::test]
16323    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
16324        let supervisor =
16325            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16326                .with_cgroup_placement(None);
16327        let result = supervisor.spawn(ModuleSpec {
16328            script: None,
16329            module_id: "no-cgroup-placement".to_string(),
16330            program: fake_aft_stub_path(),
16331            args: Vec::new(),
16332            env: Vec::new(),
16333            reserved: false,
16334            reserved_prefixes: Vec::new(),
16335            protocol: ModuleProtocol::Subc,
16336            overlap: Default::default(),
16337            carry_containment: false,
16338        });
16339
16340        assert!(
16341            result.is_ok(),
16342            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
16343        );
16344    }
16345
16346    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16347    async fn undecided_snapshot_uses_shared_restart_predicate() {
16348        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
16349            .will_recover_after_connection_loss()
16350            .unwrap());
16351        assert!(
16352            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
16353                .will_recover_after_connection_loss()
16354                .unwrap()
16355        );
16356    }
16357
16358    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16359    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
16360        assert!(
16361            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
16362                .will_recover_after_connection_loss()
16363                .unwrap()
16364        );
16365    }
16366
16367    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16368    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
16369        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
16370            .will_recover_after_connection_loss()
16371            .unwrap());
16372        assert!(
16373            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
16374                .will_recover_after_connection_loss()
16375                .unwrap()
16376        );
16377    }
16378
16379    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16380    async fn warming_snapshot_is_limited_to_startup_phases() {
16381        for state in [
16382            ModuleState::Starting,
16383            ModuleState::Running,
16384            ModuleState::Restarting,
16385        ] {
16386            assert!(
16387                module_with_recovery_snapshot(state, true, 0)
16388                    .is_warming()
16389                    .unwrap(),
16390                "{state:?} should be warming"
16391            );
16392        }
16393        for state in [
16394            ModuleState::Unresponsive,
16395            ModuleState::Draining,
16396            ModuleState::Stopped,
16397            ModuleState::Failed,
16398            ModuleState::Disabled,
16399        ] {
16400            assert!(
16401                !module_with_recovery_snapshot(state, true, 0)
16402                    .is_warming()
16403                    .unwrap(),
16404                "{state:?} should not be warming"
16405            );
16406        }
16407    }
16408
16409    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16410    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
16411        let registry = Arc::new(Registry::default());
16412        let supervisor =
16413            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
16414        let module = supervisor
16415            .spawn(ModuleSpec {
16416                script: None,
16417                module_id: "terminal-history".to_string(),
16418                program: fake_aft_stub_path(),
16419                args: Vec::new(),
16420                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16421                reserved: false,
16422                reserved_prefixes: Vec::new(),
16423                protocol: ModuleProtocol::Subc,
16424                overlap: Default::default(),
16425                carry_containment: false,
16426            })
16427            .unwrap();
16428
16429        let deadline = Instant::now() + Duration::from_secs(5);
16430        loop {
16431            let history = module.terminal_history();
16432            if history.entries.len() == 2 {
16433                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
16434                assert_eq!(history.dropped, 0);
16435                assert_eq!(
16436                    history
16437                        .entries
16438                        .iter()
16439                        .map(|entry| entry.exit_code)
16440                        .collect::<Vec<_>>(),
16441                    vec![Some(23), Some(23)]
16442                );
16443                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
16444                return;
16445            }
16446            assert!(
16447                Instant::now() < deadline,
16448                "module did not retain two terminal exits: {history:?}"
16449            );
16450            sleep(Duration::from_millis(10)).await;
16451        }
16452    }
16453
16454    /// A disable issued while a crash respawn is still backing off must preempt
16455    /// that respawn: the operator's stop wins, the disable must not queue behind
16456    /// the backoff, and the module must never come back up afterwards.
16457    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16458    async fn disable_during_crash_backoff_cancels_pending_respawn() {
16459        let backoff = Duration::from_secs(2);
16460        let supervisor = Supervisor::new_for_test(
16461            Arc::new(Registry::default()),
16462            RestartPolicy::new(10, backoff),
16463        );
16464        let module = supervisor
16465            .spawn(ModuleSpec {
16466                script: None,
16467                module_id: "disable-during-backoff".to_string(),
16468                program: fake_aft_stub_path(),
16469                args: Vec::new(),
16470                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16471                reserved: false,
16472                reserved_prefixes: Vec::new(),
16473                protocol: ModuleProtocol::Subc,
16474                overlap: Default::default(),
16475                carry_containment: false,
16476            })
16477            .unwrap();
16478
16479        // Wait for the first crash to put the module into its backoff window.
16480        let deadline = Instant::now() + Duration::from_secs(5);
16481        loop {
16482            if module.status().unwrap().state == ModuleState::Restarting {
16483                break;
16484            }
16485            assert!(
16486                Instant::now() < deadline,
16487                "module never entered the crash backoff"
16488            );
16489            sleep(Duration::from_millis(10)).await;
16490        }
16491
16492        let started = Instant::now();
16493        module.set_enabled(false).await.unwrap();
16494        let waited = started.elapsed();
16495
16496        assert!(
16497            waited < backoff / 2,
16498            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
16499        );
16500        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
16501
16502        // Outlast the backoff: the respawn it was counting down to must never run.
16503        sleep(backoff + Duration::from_millis(500)).await;
16504        let status = module.status().unwrap();
16505        assert_eq!(status.state, ModuleState::Disabled);
16506        assert_eq!(
16507            status.spawn_generation, 1,
16508            "module respawned after the operator disabled it"
16509        );
16510    }
16511
16512    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
16513    /// the shape of nats-server, the program this rule exists for.
16514    #[cfg(unix)]
16515    fn protocol_none_sigterm_exits_clean_spec(
16516        module_id: &str,
16517        dir: &std::path::Path,
16518    ) -> (ModuleSpec, PathBuf, PathBuf) {
16519        let ready = dir.join("ready");
16520        let marker = dir.join("sigterm");
16521        let spec = ModuleSpec {
16522            script: None,
16523            module_id: module_id.to_string(),
16524            program: fake_aft_stub_path(),
16525            args: Vec::new(),
16526            env: vec![
16527                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16528                (
16529                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
16530                    marker.display().to_string(),
16531                ),
16532                (
16533                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
16534                    ready.display().to_string(),
16535                ),
16536            ],
16537            reserved: false,
16538            reserved_prefixes: Vec::new(),
16539            protocol: ModuleProtocol::None,
16540            overlap: Default::default(),
16541            carry_containment: false,
16542        };
16543        (spec, ready, marker)
16544    }
16545
16546    /// Wait for a file the child writes, so a signal is never sent before the
16547    /// child's SIGTERM handler is installed (the default disposition would
16548    /// kill it by signal and the exit would not be clean).
16549    #[cfg(unix)]
16550    async fn wait_for_file(path: &std::path::Path) {
16551        let deadline = Instant::now() + Duration::from_secs(10);
16552        while !path.exists() {
16553            assert!(
16554                Instant::now() < deadline,
16555                "{} never appeared",
16556                path.display()
16557            );
16558            sleep(Duration::from_millis(10)).await;
16559        }
16560    }
16561
16562    /// A protocol-none module that exits 0 because something OUTSIDE the
16563    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
16564    /// the crash-path disposition rather than `stopped`.
16565    #[cfg(unix)]
16566    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16567    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
16568        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
16569        let (spec, ready, marker) =
16570            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
16571        let supervisor = Supervisor::new_for_test(
16572            Arc::new(Registry::default()),
16573            RestartPolicy::new(3, Duration::ZERO),
16574        );
16575        let module = supervisor.spawn(spec).unwrap();
16576        wait_for_file(&ready).await;
16577        // The ready file proves the child installed its SIGTERM handler, not
16578        // that the supervisor has processed the privacy trampoline's exec
16579        // acknowledgement. On macOS status withholds the pid until then.
16580        let deadline = Instant::now() + Duration::from_secs(10);
16581        let first_pid = loop {
16582            let status = module.status().unwrap();
16583            if status.state == ModuleState::Running {
16584                if let Some(pid) = status.pid {
16585                    break pid;
16586                }
16587            }
16588            assert!(
16589                Instant::now() < deadline,
16590                "a running module must report its pid after exec confirmation: {status:?}"
16591            );
16592            sleep(Duration::from_millis(10)).await;
16593        };
16594
16595        rustix::process::kill_process(
16596            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
16597            rustix::process::Signal::TERM,
16598        )
16599        .unwrap();
16600
16601        let deadline = Instant::now() + Duration::from_secs(10);
16602        let respawned = loop {
16603            let status = module.status().unwrap();
16604            if status.state == ModuleState::Running
16605                && status.pid.is_some_and(|pid| pid != first_pid)
16606            {
16607                break status;
16608            }
16609            assert!(
16610                Instant::now() < deadline,
16611                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
16612            );
16613            sleep(Duration::from_millis(10)).await;
16614        };
16615        assert_eq!(respawned.spawn_generation, 2);
16616        assert!(
16617            marker.exists(),
16618            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
16619        );
16620
16621        let history = module.terminal_history();
16622        assert_eq!(history.entries.len(), 1, "{history:?}");
16623        let entry = &history.entries[0];
16624        assert_eq!(entry.exit_code, Some(0));
16625        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
16626        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
16627
16628        module.stop().await.unwrap();
16629    }
16630
16631    /// Repeated unrequested clean exits of a protocol-none module spend the
16632    /// restart budget exactly as crashes do, and the module ends `failed` with
16633    /// the budget named.
16634    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16635    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
16636        let supervisor = Supervisor::new_for_test(
16637            Arc::new(Registry::default()),
16638            RestartPolicy::new(1, Duration::ZERO),
16639        );
16640        let module = supervisor
16641            .spawn(ModuleSpec {
16642                script: None,
16643                module_id: "none-clean-exit-budget".to_string(),
16644                program: fake_aft_stub_path(),
16645                args: Vec::new(),
16646                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
16647                reserved: false,
16648                reserved_prefixes: Vec::new(),
16649                protocol: ModuleProtocol::None,
16650                overlap: Default::default(),
16651                carry_containment: false,
16652            })
16653            .unwrap();
16654
16655        // Failed follows the terminal write; this deadline only bounds a hang,
16656        // not an assumed duration for the two launches or their exit recording.
16657        let deadline = Instant::now() + Duration::from_secs(10);
16658        loop {
16659            let status = module.status().unwrap();
16660            if status.state == ModuleState::Failed {
16661                break;
16662            }
16663            assert!(
16664                Instant::now() < deadline,
16665                "module never exhausted its budget: {status:?} {:?}",
16666                module.terminal_history()
16667            );
16668            sleep(Duration::from_millis(10)).await;
16669        }
16670        let history = module.terminal_history();
16671        assert_eq!(
16672            history
16673                .entries
16674                .iter()
16675                .map(|entry| (entry.exit_code, entry.disposition.clone()))
16676                .collect::<Vec<_>>(),
16677            vec![
16678                (Some(0), TerminalDisposition::Restarting),
16679                (Some(0), TerminalDisposition::Failed),
16680            ]
16681        );
16682        let detail = history.entries[1]
16683            .disposition_detail
16684            .as_deref()
16685            .expect("a budget failure names the budget");
16686        assert!(detail.contains("max_restarts=1"), "{detail}");
16687        assert_eq!(module.status().unwrap().spawn_generation, 2);
16688    }
16689
16690    #[test]
16691    fn restart_budget_failure_is_published_after_its_terminal_record() {
16692        use std::io::{Read, Write};
16693        let supervisor = Supervisor::new_for_test(
16694            Arc::new(Registry::default()),
16695            RestartPolicy::new(0, Duration::ZERO),
16696        );
16697        let runtime = supervisor.runtime_config();
16698        let spec = ModuleSpec {
16699            script: None,
16700            protocol: ModuleProtocol::None,
16701            ..windowed_crash_spec("budget-publication-order")
16702        };
16703        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
16704            ModuleState::Running,
16705            true,
16706        )));
16707        lock_snapshot(&snapshot).unwrap().process_alive = true;
16708        let reaped_snapshot = snapshot.clone();
16709        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16710        let history = runtime.terminal_ring.clone();
16711        let reap = std::thread::spawn(move || {
16712            tokio::runtime::Builder::new_current_thread()
16713                .enable_all()
16714                .build()
16715                .unwrap()
16716                .block_on(on_child_exit(
16717                    &spec,
16718                    runtime.restart_policy,
16719                    &supervisor.registry,
16720                    &reaped_snapshot,
16721                    &runtime.terminal_ring,
16722                    &runtime.spawn_events,
16723                    &runtime.child_roster,
16724                    ExitReport {
16725                        kind: ExitKind::Clean,
16726                        code: Some(0),
16727                        signal: None,
16728                        at_ms: 1,
16729                    },
16730                    true,
16731                ))
16732        });
16733        let mut reached = [0];
16734        barrier.read_exact(&mut reached).unwrap();
16735        assert_eq!(&reached, b"R");
16736        let (before_state, before_alive) = {
16737            let state = lock_snapshot(&snapshot).unwrap();
16738            (state.state, state.process_alive)
16739        };
16740        let before_history = history.lock().unwrap().snapshot();
16741        barrier.write_all(b"X").unwrap();
16742        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
16743        assert!(before_history.entries.is_empty());
16744        assert_ne!(
16745            before_state,
16746            ModuleState::Failed,
16747            "Failed was visible before its terminal record could be written"
16748        );
16749        assert!(before_alive);
16750        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
16751        let history = history.lock().unwrap().snapshot();
16752        assert_eq!(history.entries.len(), 1);
16753        assert_eq!(history.entries[0].exit_code, Some(0));
16754        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16755    }
16756
16757    #[test]
16758    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
16759        use std::io::{Read, Write};
16760
16761        let supervisor = Supervisor::new_for_test(
16762            Arc::new(Registry::default()),
16763            RestartPolicy::new(3, Duration::ZERO),
16764        );
16765        let mut runtime = supervisor.runtime_config();
16766        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
16767        let snapshot = module.inner.snapshot.clone();
16768        lock_snapshot(&snapshot).unwrap().process_alive = true;
16769        runtime.terminal_ring = module.inner.terminal_ring.clone();
16770        let spec = windowed_crash_spec(module.module_id());
16771        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16772        let reaped_snapshot = snapshot.clone();
16773        let reap = std::thread::spawn(move || {
16774            tokio::runtime::Builder::new_current_thread()
16775                .enable_all()
16776                .build()
16777                .unwrap()
16778                .block_on(on_child_exit(
16779                    &spec,
16780                    runtime.restart_policy,
16781                    &supervisor.registry,
16782                    &reaped_snapshot,
16783                    &runtime.terminal_ring,
16784                    &runtime.spawn_events,
16785                    &runtime.child_roster,
16786                    ExitReport {
16787                        kind: ExitKind::Crash,
16788                        code: Some(2),
16789                        signal: None,
16790                        at_ms: 1,
16791                    },
16792                    true,
16793                ))
16794        });
16795
16796        // Hold the crash handler between its budget decision and publication.
16797        // Connection cleanup must still see the recovery that this crash chose,
16798        // even when that recovery consumes the last available restart slot.
16799        let mut reached = [0];
16800        barrier.read_exact(&mut reached).unwrap();
16801        assert_eq!(&reached, b"R");
16802        let before_status = module.status().unwrap();
16803        let before_history = module.terminal_history();
16804        let before_recovers = module.will_recover_after_connection_loss().unwrap();
16805        barrier.write_all(b"X").unwrap();
16806        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
16807
16808        assert_eq!(before_status.state, ModuleState::Running);
16809        assert!(before_status.process_alive);
16810        assert_eq!(before_status.restart_count, 3);
16811        assert!(before_history.entries.is_empty());
16812        assert!(
16813            before_recovers,
16814            "connection cleanup would report a terminal close for a scheduled crash restart"
16815        );
16816        let restarting = module.status().unwrap();
16817        assert_eq!(restarting.state, ModuleState::Restarting);
16818        assert_eq!(restarting.restart_count, 3);
16819        assert!(module.will_recover_after_connection_loss().unwrap());
16820        let history = module.terminal_history();
16821        assert_eq!(history.entries.len(), 1);
16822        assert_eq!(
16823            history.entries[0].disposition,
16824            TerminalDisposition::Restarting
16825        );
16826
16827        // Once the replacement is running, its next crash must be judged
16828        // against the exhausted budget rather than the previous recovery.
16829        update_snapshot(&snapshot, Some(module.module_id()), |state| {
16830            state.state = ModuleState::Running;
16831            state.respawn_pending = false;
16832        })
16833        .unwrap();
16834        assert!(!module.will_recover_after_connection_loss().unwrap());
16835        update_snapshot(&snapshot, Some(module.module_id()), |state| {
16836            state.enabled = false;
16837            state.respawn_pending = true;
16838        })
16839        .unwrap();
16840        assert!(!module.will_recover_after_connection_loss().unwrap());
16841    }
16842
16843    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
16844        use std::net::{TcpListener, TcpStream};
16845        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
16846        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
16847        let (barrier, _) = listener.accept().unwrap();
16848        for stream in [&writer, &barrier] {
16849            stream
16850                .set_read_timeout(Some(Duration::from_secs(10)))
16851                .unwrap();
16852            stream
16853                .set_write_timeout(Some(Duration::from_secs(10)))
16854                .unwrap();
16855        }
16856        *events.1.lock().unwrap() = Some(writer);
16857        barrier
16858    }
16859
16860    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
16861        use std::io::{Read, Write};
16862        let barrier = events.1.lock().unwrap().take();
16863        if let Some(mut barrier) = barrier {
16864            barrier.write_all(b"R").unwrap();
16865            let mut release = [0];
16866            barrier.read_exact(&mut release).unwrap();
16867            assert_eq!(&release, b"X");
16868        }
16869    }
16870
16871    #[test]
16872    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
16873        use std::io::{Read, Write};
16874
16875        let supervisor = Supervisor::new_for_test(
16876            Arc::new(Registry::default()),
16877            RestartPolicy::new(0, Duration::ZERO),
16878        );
16879        let mut runtime = supervisor.runtime_config();
16880        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
16881        let snapshot = module.inner.snapshot.clone();
16882        runtime.terminal_ring = module.inner.terminal_ring.clone();
16883        let spec = ModuleSpec {
16884            script: None,
16885            module_id: module.module_id().into(),
16886            program: fake_aft_stub_path(),
16887            protocol: ModuleProtocol::None,
16888            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
16889            ..windowed_crash_spec(module.module_id())
16890        };
16891        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16892        let reap = std::thread::spawn(move || {
16893            tokio::runtime::Builder::new_current_thread()
16894                .enable_all()
16895                .build()
16896                .unwrap()
16897                .block_on(async {
16898                    let mut child =
16899                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
16900                    health_restart_child(
16901                        &spec,
16902                        &runtime,
16903                        &supervisor.registry,
16904                        &supervisor.process_liveness,
16905                        &snapshot,
16906                        &mut child,
16907                        SupervisorHealthStatus::Failing,
16908                        Some("health probe did not answer"),
16909                        1,
16910                    )
16911                    .await
16912                    .unwrap();
16913                    assert!(child.is_none());
16914                });
16915        });
16916        // The loopback barrier acknowledges entry into terminal recording by
16917        // health_restart_child's drain, then waits for the test's release byte.
16918        // Status and history are observed while that writer is held, not after
16919        // an elapsed-time guess about when the health action has completed.
16920        let mut reached = [0];
16921        barrier.read_exact(&mut reached).unwrap();
16922        assert_eq!(&reached, b"R");
16923        let before_status = module.status().unwrap();
16924        let before_history = module.terminal_history();
16925        barrier.write_all(b"X").unwrap();
16926        reap.join().unwrap();
16927        assert!(before_history.entries.is_empty());
16928        assert_ne!(
16929            before_status.state,
16930            ModuleState::Failed,
16931            "Failed was visible before its terminal record could be written"
16932        );
16933        assert!(
16934            before_status.process_alive,
16935            "process death was visible before its terminal record could be written"
16936        );
16937        let failed = module.status().unwrap();
16938        assert_eq!(failed.state, ModuleState::Failed);
16939        assert!(!failed.process_alive);
16940        let history = module.terminal_history();
16941        assert_eq!(history.entries.len(), 1);
16942        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16943        assert_eq!(
16944            history.entries[0].disposition_detail.as_deref(),
16945            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
16946        );
16947    }
16948
16949    #[test]
16950    fn observed_exits_publish_process_death_after_terminal_history() {
16951        use std::io::{Read, Write};
16952        for (kind, enabled, shutdown, wait_error, expected_state) in [
16953            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
16954            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
16955            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
16956            (
16957                ExitKind::DeliberateSeverance,
16958                true,
16959                false,
16960                false,
16961                ModuleState::Restarting,
16962            ),
16963            #[cfg(unix)]
16964            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
16965            (ExitKind::Crash, true, false, true, ModuleState::Failed),
16966        ] {
16967            let supervisor = Supervisor::new_for_test(
16968                Arc::new(Registry::default()),
16969                RestartPolicy::new(1, Duration::ZERO),
16970            );
16971            let runtime = supervisor.runtime_config();
16972            #[cfg(unix)]
16973            if shutdown {
16974                runtime.child_roster.close();
16975            }
16976            #[cfg(not(unix))]
16977            let _ = shutdown;
16978            let spec = windowed_crash_spec("exit-publication-order");
16979            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
16980                ModuleState::Running,
16981                enabled,
16982            )));
16983            lock_snapshot(&snapshot).unwrap().process_alive = true;
16984            let reaped_snapshot = snapshot.clone();
16985            let history = runtime.terminal_ring.clone();
16986            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16987            let reap = std::thread::spawn(move || {
16988                if wait_error {
16989                    super::fail_after_wait_error(
16990                        &spec.module_id,
16991                        &reaped_snapshot,
16992                        &runtime.terminal_ring,
16993                        &runtime.spawn_events,
16994                        "wait failed".into(),
16995                        &runtime.failure_context(true),
16996                    );
16997                } else {
16998                    tokio::runtime::Builder::new_current_thread()
16999                        .enable_all()
17000                        .build()
17001                        .unwrap()
17002                        .block_on(on_child_exit(
17003                            &spec,
17004                            runtime.restart_policy,
17005                            &supervisor.registry,
17006                            &reaped_snapshot,
17007                            &runtime.terminal_ring,
17008                            &runtime.spawn_events,
17009                            &runtime.child_roster,
17010                            ExitReport {
17011                                kind,
17012                                code: Some(0),
17013                                signal: None,
17014                                at_ms: 1,
17015                            },
17016                            true,
17017                        ));
17018                }
17019            });
17020            let mut reached = [0];
17021            barrier.read_exact(&mut reached).unwrap();
17022            assert_eq!(&reached, b"R");
17023            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
17024            let before_history = history.lock().unwrap().snapshot();
17025            barrier.write_all(b"X").unwrap();
17026            reap.join().unwrap();
17027            assert!(
17028                before_alive,
17029                "{kind:?}: death was published before recording"
17030            );
17031            assert!(before_history.entries.is_empty());
17032            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
17033            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
17034            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
17035        }
17036    }
17037
17038    #[tokio::test]
17039    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
17040        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
17041        let path = dir.join("terminals.jsonl");
17042        std::fs::create_dir(&path).unwrap();
17043        let supervisor = Supervisor::new_for_test(
17044            Arc::new(Registry::default()),
17045            RestartPolicy::new(0, Duration::ZERO),
17046        )
17047        .with_terminal_journal(path, "health-budget-journal-failure".into());
17048        let runtime = supervisor.runtime_config();
17049        let spec = ModuleSpec {
17050            script: None,
17051            program: fake_aft_stub_path(),
17052            protocol: ModuleProtocol::None,
17053            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
17054            ..windowed_crash_spec("health-budget-journal-failure")
17055        };
17056        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17057        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
17058        health_restart_child(
17059            &spec,
17060            &runtime,
17061            &supervisor.registry,
17062            &supervisor.process_liveness,
17063            &snapshot,
17064            &mut child,
17065            SupervisorHealthStatus::Failing,
17066            None,
17067            1,
17068        )
17069        .await
17070        .unwrap();
17071        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17072        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
17073        let history = runtime
17074            .terminal_ring
17075            .lock()
17076            .unwrap()
17077            .durable_history(&spec.module_id);
17078        assert!(history.journal_write_failures > 0);
17079        assert_eq!(history.entries.len(), 1);
17080        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
17081    }
17082
17083    #[test]
17084    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
17085        let supervisor = Supervisor::new_for_test(
17086            Arc::new(Registry::default()),
17087            RestartPolicy::new(0, Duration::ZERO),
17088        );
17089        let runtime = supervisor.runtime_config();
17090        let spec = ModuleSpec {
17091            script: None,
17092            program: fake_aft_stub_path(),
17093            protocol: ModuleProtocol::None,
17094            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
17095            ..windowed_crash_spec("health-budget-recording-panic")
17096        };
17097        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17098        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
17099            tokio::runtime::Builder::new_current_thread()
17100                .enable_all()
17101                .build()
17102                .unwrap()
17103                .block_on(async {
17104                    let mut child =
17105                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
17106                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
17107                    health_restart_child(
17108                        &spec,
17109                        &runtime,
17110                        &supervisor.registry,
17111                        &supervisor.process_liveness,
17112                        &snapshot,
17113                        &mut child,
17114                        SupervisorHealthStatus::Failing,
17115                        None,
17116                        1,
17117                    )
17118                    .await
17119                    .unwrap();
17120                });
17121        }));
17122        let panic = result.expect_err("recording must still unwind");
17123        assert_eq!(
17124            panic.downcast_ref::<String>().map(String::as_str),
17125            Some("spawn event sequence exhausted")
17126        );
17127        let state = lock_snapshot(&snapshot).unwrap();
17128        assert_eq!(state.state, ModuleState::Failed);
17129        assert!(!state.process_alive);
17130        assert!(state.last_exit.is_some());
17131    }
17132
17133    #[tokio::test]
17134    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
17135        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
17136        let path = dir.join("terminals.jsonl");
17137        std::fs::create_dir(&path).unwrap();
17138        let supervisor = Supervisor::new_for_test(
17139            Arc::new(Registry::default()),
17140            RestartPolicy::new(0, Duration::ZERO),
17141        )
17142        .with_terminal_journal(path, "budget-journal-failure".into());
17143        let runtime = supervisor.runtime_config();
17144        let spec = windowed_crash_spec("budget-journal-failure");
17145        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17146        assert!(matches!(
17147            on_child_exit(
17148                &spec,
17149                runtime.restart_policy,
17150                &supervisor.registry,
17151                &snapshot,
17152                &runtime.terminal_ring,
17153                &runtime.spawn_events,
17154                &runtime.child_roster,
17155                crash_exit_report(1),
17156                true,
17157            )
17158            .await,
17159            NextAction::Stop { .. }
17160        ));
17161        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17162        let history = runtime
17163            .terminal_ring
17164            .lock()
17165            .unwrap()
17166            .durable_history(&spec.module_id);
17167        assert!(history.journal_write_failures > 0);
17168        assert_eq!(history.entries.len(), 1);
17169        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
17170    }
17171
17172    #[test]
17173    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
17174        let supervisor = Supervisor::new_for_test(
17175            Arc::new(Registry::default()),
17176            RestartPolicy::new(0, Duration::ZERO),
17177        );
17178        let runtime = supervisor.runtime_config();
17179        let spec = windowed_crash_spec("budget-recording-panic");
17180        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17181        runtime
17182            .spawn_events
17183            .emit_spawned(&spec.module_id, None, 1, 1);
17184        // Exhausting the event sequence makes emit_exited panic before the
17185        // terminal write, exercising publication on the recording unwind.
17186        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
17187        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
17188            tokio::runtime::Builder::new_current_thread()
17189                .enable_all()
17190                .build()
17191                .unwrap()
17192                .block_on(on_child_exit(
17193                    &spec,
17194                    runtime.restart_policy,
17195                    &supervisor.registry,
17196                    &snapshot,
17197                    &runtime.terminal_ring,
17198                    &runtime.spawn_events,
17199                    &runtime.child_roster,
17200                    crash_exit_report(1),
17201                    true,
17202                ))
17203        }));
17204        let panic = result.err().expect("recording must still unwind");
17205        assert_eq!(
17206            panic.downcast_ref::<String>().map(String::as_str),
17207            Some("spawn event sequence exhausted")
17208        );
17209        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17210    }
17211
17212    /// A stop the supervisor itself requests still stops a protocol-none
17213    /// module, even though the child answers the SIGTERM with exit 0.
17214    #[cfg(unix)]
17215    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17216    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
17217        for disable in [false, true] {
17218            let label = if disable {
17219                "none-requested-disable"
17220            } else {
17221                "none-requested-stop"
17222            };
17223            let dir = cortexkit_test_support::ScratchDir::new(label);
17224            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
17225            let supervisor = Supervisor::new_for_test(
17226                Arc::new(Registry::default()),
17227                RestartPolicy::new(3, Duration::ZERO),
17228            );
17229            let module = supervisor.spawn(spec).unwrap();
17230            wait_for_file(&ready).await;
17231
17232            if disable {
17233                module.set_enabled(false).await.unwrap();
17234            } else {
17235                module.stop().await.unwrap();
17236            }
17237            assert!(
17238                marker.exists(),
17239                "{label}: the child must have left through its SIGTERM handler with exit 0"
17240            );
17241
17242            // Long enough for a zero-backoff respawn to have happened if the
17243            // exit had been treated as a crash.
17244            sleep(Duration::from_millis(500)).await;
17245            let status = module.status().unwrap();
17246            let expected = if disable {
17247                ModuleState::Disabled
17248            } else {
17249                ModuleState::Stopped
17250            };
17251            assert_eq!(status.state, expected, "{label}");
17252            assert_eq!(
17253                status.spawn_generation, 1,
17254                "{label}: respawned after a requested stop"
17255            );
17256            let history = module.terminal_history();
17257            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
17258            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
17259            assert_ne!(
17260                history.entries[0].disposition,
17261                TerminalDisposition::Restarting,
17262                "{label}"
17263            );
17264        }
17265    }
17266
17267    /// A subc-wire module that exits 0 on its own is still a stop: the
17268    /// protocol-none rule must not reach it.
17269    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17270    async fn subc_wire_clean_exit_is_still_a_stop() {
17271        let supervisor = Supervisor::new_for_test(
17272            Arc::new(Registry::default()),
17273            RestartPolicy::new(3, Duration::ZERO),
17274        );
17275        let module = supervisor
17276            .spawn(ModuleSpec {
17277                script: None,
17278                module_id: "wire-clean-exit".to_string(),
17279                program: fake_aft_stub_path(),
17280                args: Vec::new(),
17281                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
17282                reserved: false,
17283                reserved_prefixes: Vec::new(),
17284                protocol: ModuleProtocol::Subc,
17285                overlap: Default::default(),
17286                carry_containment: false,
17287            })
17288            .unwrap();
17289
17290        let deadline = Instant::now() + Duration::from_secs(10);
17291        while module.terminal_history().entries.is_empty() {
17292            assert!(Instant::now() < deadline, "module never exited");
17293            sleep(Duration::from_millis(10)).await;
17294        }
17295        // Long enough for a zero-backoff respawn to have happened.
17296        sleep(Duration::from_millis(500)).await;
17297        let status = module.status().unwrap();
17298        assert_eq!(status.state, ModuleState::Stopped);
17299        assert_eq!(status.spawn_generation, 1);
17300        let history = module.terminal_history();
17301        assert_eq!(history.entries.len(), 1, "{history:?}");
17302        assert_eq!(history.entries[0].exit_code, Some(0));
17303        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
17304    }
17305
17306    #[cfg(unix)]
17307    #[tokio::test]
17308    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
17309        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
17310        let record = dir.join("live-children.json");
17311        let supervisor = Supervisor::new_for_test(
17312            Arc::new(Registry::default()),
17313            RestartPolicy::new(0, Duration::ZERO),
17314        );
17315        let mut runtime = supervisor.runtime_config();
17316        runtime.child_roster.record_to(record.clone());
17317        let gate = Arc::new(super::ReloadExitRecordGate::default());
17318        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
17319        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17320        let spec = ModuleSpec {
17321            script: None,
17322            module_id: "reload-exit-roster".into(),
17323            program: fake_aft_stub_path(),
17324            args: Vec::new(),
17325            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
17326            reserved: false,
17327            reserved_prefixes: Vec::new(),
17328            protocol: ModuleProtocol::Subc,
17329            overlap: Default::default(),
17330            carry_containment: false,
17331        };
17332        let mut child = None;
17333        let reload = super::finish_reload_child(
17334            &spec,
17335            &runtime,
17336            &supervisor.registry,
17337            &supervisor.process_liveness,
17338            &snapshot,
17339            &mut child,
17340        );
17341        tokio::pin!(reload);
17342        tokio::select! {
17343            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
17344            _ = gate.reached.notified() => {}
17345        }
17346        assert!(runtime
17347            .terminal_ring
17348            .lock()
17349            .unwrap()
17350            .snapshot()
17351            .entries
17352            .is_empty());
17353        assert_eq!(
17354            crate::live_children::read_record(&record).unwrap().len(),
17355            1,
17356            "shutdown must still wait for the reaped child until its terminal record exists"
17357        );
17358        runtime.child_roster.close();
17359        gate.resume.notify_one();
17360        assert!(reload.await.is_err());
17361        assert!(crate::live_children::read_record(&record)
17362            .unwrap()
17363            .is_empty());
17364        let history = runtime.terminal_ring.lock().unwrap().snapshot();
17365        assert_eq!(history.entries.len(), 1);
17366        assert_eq!(
17367            history.entries[0].disposition,
17368            TerminalDisposition::DaemonShutdown
17369        );
17370    }
17371
17372    /// Each restart-producing arm has its own state transition. Keeping their
17373    /// lifetime count assertions adjacent prevents a later new arm from silently
17374    /// spending budget without recording the historical restart.
17375    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17376    async fn every_restart_increment_path_advances_lifetime_count() {
17377        let supervisor = Supervisor::new_for_test(
17378            Arc::new(Registry::default()),
17379            RestartPolicy::new(1, Duration::ZERO),
17380        );
17381        let runtime = supervisor.runtime_config();
17382        let spec = ModuleSpec {
17383            script: None,
17384            module_id: "lifetime-increment-path".to_string(),
17385            program: PathBuf::from("/unused/lifetime-increment-path"),
17386            args: Vec::new(),
17387            env: Vec::new(),
17388            reserved: false,
17389            reserved_prefixes: Vec::new(),
17390            protocol: ModuleProtocol::Subc,
17391            overlap: Default::default(),
17392            carry_containment: false,
17393        };
17394
17395        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17396        assert!(matches!(
17397            on_child_exit(
17398                &spec,
17399                runtime.restart_policy,
17400                &supervisor.registry,
17401                &crash_snapshot,
17402                &runtime.terminal_ring,
17403                &runtime.spawn_events,
17404                &runtime.child_roster,
17405                ExitReport {
17406                    kind: ExitKind::Crash,
17407                    code: Some(1),
17408                    signal: None,
17409                    at_ms: 1,
17410                },
17411                true,
17412            )
17413            .await,
17414            NextAction::Restart { schedule: _ }
17415        ));
17416        let (crash_restarts, crash_lifetime) = {
17417            let state = lock_snapshot(&crash_snapshot).unwrap();
17418            (state.crash_restarts.len(), state.lifetime_restarts)
17419        };
17420        assert_eq!(crash_restarts, 1);
17421        assert_eq!(crash_lifetime, 1);
17422
17423        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17424        let mut health_child = None;
17425        assert!(matches!(
17426            health_restart_child(
17427                &spec,
17428                &runtime,
17429                &supervisor.registry,
17430                &supervisor.process_liveness,
17431                &health_snapshot,
17432                &mut health_child,
17433                SupervisorHealthStatus::Failing,
17434                None,
17435                2,
17436            )
17437            .await,
17438            Ok(())
17439        ));
17440        assert!(health_child.is_none());
17441        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
17442        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
17443        let (health_restarts, health_lifetime) = {
17444            let state = lock_snapshot(&health_snapshot).unwrap();
17445            (state.crash_restarts.len(), state.lifetime_restarts)
17446        };
17447        assert_eq!(health_restarts, 1);
17448        assert_eq!(health_lifetime, 1);
17449
17450        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17451        let mut reload_child = None;
17452        assert!(matches!(
17453            handle_reload_spawn_failure(
17454                &spec,
17455                &runtime,
17456                &supervisor.process_liveness,
17457                &reload_snapshot,
17458                &mut reload_child,
17459                "forced reload spawn failure".to_string(),
17460            )
17461            .await,
17462            Err(SuperviseError::ReloadFailed { .. })
17463        ));
17464        let (reload_restarts, reload_lifetime) = {
17465            let state = lock_snapshot(&reload_snapshot).unwrap();
17466            (state.crash_restarts.len(), state.lifetime_restarts)
17467        };
17468        assert_eq!(reload_restarts, 1);
17469        assert_eq!(reload_lifetime, 1);
17470    }
17471
17472    #[tokio::test]
17473    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
17474        let supervisor = Supervisor::new_for_test(
17475            Arc::new(Registry::default()),
17476            RestartPolicy::new(3, Duration::ZERO),
17477        );
17478        let runtime = supervisor.runtime_config();
17479        let spec = ModuleSpec {
17480            script: None,
17481            module_id: "deliberately-severed".to_string(),
17482            program: PathBuf::from("/unused/deliberately-severed"),
17483            args: Vec::new(),
17484            env: Vec::new(),
17485            reserved: false,
17486            reserved_prefixes: Vec::new(),
17487            protocol: ModuleProtocol::Subc,
17488            overlap: Default::default(),
17489            carry_containment: false,
17490        };
17491        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17492        let process = ProcessIdentity {
17493            pid: 41,
17494            start_time: 101,
17495        };
17496        record_deliberate_severance(&snapshot, process).unwrap();
17497        let exit_report = apply_deliberate_severance_marker(
17498            &snapshot,
17499            Some(process),
17500            ExitReport {
17501                kind: ExitKind::Crash,
17502                code: Some(1),
17503                signal: None,
17504                at_ms: 1,
17505            },
17506        );
17507        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
17508
17509        assert!(matches!(
17510            on_child_exit(
17511                &spec,
17512                runtime.restart_policy,
17513                &supervisor.registry,
17514                &snapshot,
17515                &runtime.terminal_ring,
17516                &runtime.spawn_events,
17517                &runtime.child_roster,
17518                exit_report,
17519                true,
17520            )
17521            .await,
17522            NextAction::Restart { schedule: _ }
17523        ));
17524        let state = lock_snapshot(&snapshot).unwrap();
17525        assert_eq!(state.lifetime_restarts, 1);
17526        assert_eq!(state.crash_restarts.len(), 0);
17527    }
17528
17529    #[tokio::test]
17530    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
17531        let supervisor = Supervisor::new_for_test(
17532            Arc::new(Registry::default()),
17533            RestartPolicy::new(3, Duration::ZERO),
17534        );
17535        let runtime = supervisor.runtime_config();
17536        let spec = ModuleSpec {
17537            script: None,
17538            module_id: "genuine-crash".to_string(),
17539            program: PathBuf::from("/unused/genuine-crash"),
17540            args: Vec::new(),
17541            env: Vec::new(),
17542            reserved: false,
17543            reserved_prefixes: Vec::new(),
17544            protocol: ModuleProtocol::Subc,
17545            overlap: Default::default(),
17546            carry_containment: false,
17547        };
17548        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17549
17550        assert!(matches!(
17551            on_child_exit(
17552                &spec,
17553                runtime.restart_policy,
17554                &supervisor.registry,
17555                &snapshot,
17556                &runtime.terminal_ring,
17557                &runtime.spawn_events,
17558                &runtime.child_roster,
17559                ExitReport {
17560                    kind: ExitKind::Crash,
17561                    code: Some(1),
17562                    signal: None,
17563                    at_ms: 1,
17564                },
17565                true,
17566            )
17567            .await,
17568            NextAction::Restart { schedule: _ }
17569        ));
17570        let state = lock_snapshot(&snapshot).unwrap();
17571        assert_eq!(state.lifetime_restarts, 1);
17572        assert_eq!(state.crash_restarts.len(), 1);
17573    }
17574
17575    fn crash_exit_report(at_ms: u64) -> ExitReport {
17576        ExitReport {
17577            kind: ExitKind::Crash,
17578            code: Some(1),
17579            signal: None,
17580            at_ms,
17581        }
17582    }
17583
17584    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
17585        ModuleSpec {
17586            script: None,
17587            module_id: module_id.to_string(),
17588            program: PathBuf::from("/unused").join(module_id),
17589            args: Vec::new(),
17590            env: Vec::new(),
17591            reserved: false,
17592            reserved_prefixes: Vec::new(),
17593            protocol: ModuleProtocol::Subc,
17594            overlap: Default::default(),
17595            carry_containment: false,
17596        }
17597    }
17598
17599    /// A real crash loop still stops. Three crashes with nothing aging out spend
17600    /// a budget of two and the third respawn is refused, and both surfaces an
17601    /// operator has -- the log line and the retained terminal record -- name the
17602    /// window rather than only the cap, because `max_restarts=2` alone is what
17603    /// this budget used to mean.
17604    #[tokio::test]
17605    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
17606        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
17607        let supervisor = Supervisor::new_for_test(
17608            Arc::new(Registry::default()),
17609            RestartPolicy::new(2, Duration::ZERO),
17610        );
17611        let runtime = supervisor.runtime_config();
17612        let spec = windowed_crash_spec("crash-loop-in-window");
17613        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17614
17615        for attempt in 1..=2 {
17616            assert!(
17617                matches!(
17618                    on_child_exit(
17619                        &spec,
17620                        runtime.restart_policy,
17621                        &supervisor.registry,
17622                        &snapshot,
17623                        &runtime.terminal_ring,
17624                        &runtime.spawn_events,
17625                        &runtime.child_roster,
17626                        crash_exit_report(attempt),
17627                        true,
17628                    )
17629                    .await,
17630                    NextAction::Restart { schedule: _ }
17631                ),
17632                "crash {attempt} is inside the budget and must respawn"
17633            );
17634        }
17635
17636        assert!(matches!(
17637            on_child_exit(
17638                &spec,
17639                runtime.restart_policy,
17640                &supervisor.registry,
17641                &snapshot,
17642                &runtime.terminal_ring,
17643                &runtime.spawn_events,
17644                &runtime.child_roster,
17645                crash_exit_report(3),
17646                true,
17647            )
17648            .await,
17649            NextAction::Stop { .. }
17650        ));
17651
17652        {
17653            let state = lock_snapshot(&snapshot).unwrap();
17654            assert_eq!(state.state, ModuleState::Failed);
17655            assert_eq!(state.crash_restarts.len(), 2);
17656            assert_eq!(state.lifetime_restarts, 2);
17657        }
17658
17659        let history = runtime
17660            .terminal_ring
17661            .lock()
17662            .expect("terminal ring is not poisoned")
17663            .snapshot();
17664        let last = history
17665            .entries
17666            .last()
17667            .expect("the refused crash is retained");
17668        assert_eq!(last.disposition, TerminalDisposition::Failed);
17669        assert_eq!(
17670            last.disposition_detail.as_deref(),
17671            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
17672        );
17673
17674        let captured = crate::router::test_log::captured_logs(&logs);
17675        assert!(
17676            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
17677            "the stop must be logged with its window: {captured}"
17678        );
17679    }
17680
17681    /// The rate, stated as a test: three crashes where the first has aged past
17682    /// the window are two crashes as far as the budget is concerned, so the
17683    /// third respawn is allowed and the ring holds only the two recent ones.
17684    ///
17685    /// This is the case a lifetime counter got wrong -- and the case the daemon
17686    /// now hits routinely, since a module exits non-zero every time its
17687    /// connection to the daemon drops.
17688    #[tokio::test]
17689    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
17690        let supervisor = Supervisor::new_for_test(
17691            Arc::new(Registry::default()),
17692            RestartPolicy::new(2, Duration::ZERO),
17693        );
17694        let runtime = supervisor.runtime_config();
17695        let spec = windowed_crash_spec("crash-across-windows");
17696        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17697
17698        for attempt in 1..=2 {
17699            assert!(matches!(
17700                on_child_exit(
17701                    &spec,
17702                    runtime.restart_policy,
17703                    &supervisor.registry,
17704                    &snapshot,
17705                    &runtime.terminal_ring,
17706                    &runtime.spawn_events,
17707                    &runtime.child_roster,
17708                    crash_exit_report(attempt),
17709                    true,
17710                )
17711                .await,
17712                NextAction::Restart { schedule: _ }
17713            ));
17714        }
17715
17716        // The oldest crash moves out of the window; nothing else about the
17717        // module changes.
17718        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17719            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
17720        })
17721        .unwrap();
17722
17723        assert!(
17724            matches!(
17725                on_child_exit(
17726                    &spec,
17727                    runtime.restart_policy,
17728                    &supervisor.registry,
17729                    &snapshot,
17730                    &runtime.terminal_ring,
17731                    &runtime.spawn_events,
17732                    &runtime.child_roster,
17733                    crash_exit_report(3),
17734                    true,
17735                )
17736                .await,
17737                NextAction::Restart { schedule: _ }
17738            ),
17739            "a crash older than the window must not hold a budget slot"
17740        );
17741
17742        let state = lock_snapshot(&snapshot).unwrap();
17743        assert_eq!(state.state, ModuleState::Restarting);
17744        assert_eq!(
17745            state.crash_restarts.len(),
17746            2,
17747            "the aged instant is dropped and the new one takes its place"
17748        );
17749        assert_eq!(
17750            state.lifetime_restarts, 3,
17751            "the ledger counts every restart, including the ones the window forgot"
17752        );
17753    }
17754
17755    /// An operator restart hands the budget back whole, and the ledger keeps
17756    /// counting. Those are different questions -- "how close is this module to
17757    /// being stopped" and "how many times has it been replaced" -- and the
17758    /// operator action answers only the first.
17759    #[tokio::test]
17760    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
17761        let supervisor = Supervisor::new_for_test(
17762            Arc::new(Registry::default()),
17763            RestartPolicy::new(2, Duration::ZERO),
17764        );
17765        let runtime = supervisor.runtime_config();
17766        let spec = windowed_crash_spec("operator-cleared-budget");
17767        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17768
17769        for attempt in 1..=2 {
17770            assert!(matches!(
17771                on_child_exit(
17772                    &spec,
17773                    runtime.restart_policy,
17774                    &supervisor.registry,
17775                    &snapshot,
17776                    &runtime.terminal_ring,
17777                    &runtime.spawn_events,
17778                    &runtime.child_roster,
17779                    crash_exit_report(attempt),
17780                    true,
17781                )
17782                .await,
17783                NextAction::Restart { schedule: _ }
17784            ));
17785        }
17786
17787        reset_restart_count(&snapshot, &spec.module_id).unwrap();
17788        {
17789            let state = lock_snapshot(&snapshot).unwrap();
17790            assert!(
17791                state.crash_restarts.is_empty(),
17792                "an operator restart returns the full budget"
17793            );
17794            assert_eq!(
17795                state.lifetime_restarts, 2,
17796                "clearing the budget must not unmake the crashes"
17797            );
17798        }
17799
17800        assert!(
17801            matches!(
17802                on_child_exit(
17803                    &spec,
17804                    runtime.restart_policy,
17805                    &supervisor.registry,
17806                    &snapshot,
17807                    &runtime.terminal_ring,
17808                    &runtime.spawn_events,
17809                    &runtime.child_roster,
17810                    crash_exit_report(3),
17811                    true,
17812                )
17813                .await,
17814                NextAction::Restart { schedule: _ }
17815            ),
17816            "the cleared budget must be spendable again"
17817        );
17818        let state = lock_snapshot(&snapshot).unwrap();
17819        assert_eq!(state.crash_restarts.len(), 1);
17820        assert_eq!(state.lifetime_restarts, 3);
17821    }
17822
17823    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17824    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
17825        let severed = ProcessIdentity {
17826            pid: 41,
17827            start_time: 101,
17828        };
17829        let successor = ProcessIdentity {
17830            pid: 41,
17831            start_time: 202,
17832        };
17833        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
17834        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
17835            state.pid = Some(successor.pid);
17836            state.process_start_time = Some(successor.start_time);
17837        })
17838        .unwrap();
17839        assert!(!module.record_deliberate_severance(severed).unwrap());
17840
17841        let exit_report = apply_deliberate_severance_marker(
17842            &module.inner.snapshot,
17843            Some(successor),
17844            ExitReport {
17845                kind: ExitKind::Crash,
17846                code: Some(1),
17847                signal: None,
17848                at_ms: 1,
17849            },
17850        );
17851
17852        assert_eq!(exit_report.kind, ExitKind::Crash);
17853    }
17854
17855    #[tokio::test]
17856    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
17857        let registry = Registry::default();
17858        let supervisor = Supervisor::new_for_test(
17859            Arc::new(Registry::default()),
17860            RestartPolicy::new(3, Duration::ZERO),
17861        );
17862        let runtime = supervisor.runtime_config();
17863        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17864        let spec = ModuleSpec {
17865            script: None,
17866            module_id: "drain-deliberate-severance".to_string(),
17867            program: fake_aft_stub_path(),
17868            args: Vec::new(),
17869            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17870            reserved: false,
17871            reserved_prefixes: Vec::new(),
17872            protocol: ModuleProtocol::Subc,
17873            overlap: Default::default(),
17874            carry_containment: false,
17875        };
17876        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17877        let process = ProcessIdentity {
17878            pid: 41,
17879            start_time: 101,
17880        };
17881        child.process_identity = Some(process);
17882        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17883            state.pid = Some(process.pid);
17884            state.process_start_time = Some(process.start_time);
17885        })
17886        .unwrap();
17887        record_deliberate_severance(&snapshot, process).unwrap();
17888
17889        drain_child_to_state(
17890            &spec.module_id,
17891            spec.protocol,
17892            // The child exits on its own; no signal may change the exit this
17893            // test classifies.
17894            StopNotice::SentOverConnection,
17895            &registry,
17896            None,
17897            &snapshot,
17898            &runtime.terminal_ring,
17899            &runtime.spawn_events,
17900            child,
17901            Duration::from_secs(1),
17902            ModuleState::Stopped,
17903            Some(false),
17904            &super::FailureContext::operator(),
17905        )
17906        .await
17907        .unwrap();
17908
17909        let state = lock_snapshot(&snapshot).unwrap();
17910        assert_eq!(
17911            state.last_exit.as_ref().map(|exit| exit.kind),
17912            Some(ExitKind::DeliberateSeverance)
17913        );
17914        assert_eq!(state.lifetime_restarts, 1);
17915        assert_eq!(state.crash_restarts.len(), 0);
17916        drop(state);
17917        let history = runtime.terminal_ring.lock().unwrap().snapshot();
17918        assert_eq!(
17919            history.entries[0].exit_kind,
17920            subc_control::TerminalExitKind::DeliberateSeverance
17921        );
17922    }
17923
17924    #[tokio::test]
17925    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
17926        let registry = Registry::default();
17927        let supervisor = Supervisor::new_for_test(
17928            Arc::new(Registry::default()),
17929            RestartPolicy::new(3, Duration::ZERO),
17930        );
17931        let runtime = supervisor.runtime_config();
17932        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17933        let spec = ModuleSpec {
17934            script: None,
17935            module_id: "ordinary-drain".to_string(),
17936            program: fake_aft_stub_path(),
17937            args: Vec::new(),
17938            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17939            reserved: false,
17940            reserved_prefixes: Vec::new(),
17941            protocol: ModuleProtocol::Subc,
17942            overlap: Default::default(),
17943            carry_containment: false,
17944        };
17945        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17946
17947        drain_child_to_state(
17948            &spec.module_id,
17949            spec.protocol,
17950            // The child exits on its own; no signal may change the exit this
17951            // test classifies.
17952            StopNotice::SentOverConnection,
17953            &registry,
17954            None,
17955            &snapshot,
17956            &runtime.terminal_ring,
17957            &runtime.spawn_events,
17958            child,
17959            Duration::from_secs(1),
17960            ModuleState::Stopped,
17961            Some(false),
17962            &super::FailureContext::operator(),
17963        )
17964        .await
17965        .unwrap();
17966
17967        let state = lock_snapshot(&snapshot).unwrap();
17968        assert_eq!(
17969            state.last_exit.as_ref().map(|exit| exit.kind),
17970            Some(ExitKind::Crash)
17971        );
17972        assert_eq!(state.lifetime_restarts, 0);
17973        assert_eq!(state.crash_restarts.len(), 0);
17974    }
17975
17976    #[test]
17977    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
17978        // The server's generic fatal-routing branch only knows that the
17979        // connection failed; it does not know that the daemon deliberately
17980        // initiated a process-killing severance. Keep this seam explicit so a
17981        // future connection error path cannot silently reintroduce the stale
17982        // exemption that mislabels a later genuine crash.
17983        assert!(!include_str!("server.rs")
17984            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
17985    }
17986
17987    /// The `route.closed` `drained` value must be the quiescence wait's own
17988    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
17989    /// measurement at all and `false` is the one honest constant. This is the exact
17990    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
17991    /// on every return path, including the one that used to return early via `?`
17992    /// with `route.closing` already sent and no `route.closed` ever following.
17993    #[test]
17994    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
17995        assert!(drained_after_quiescence_wait(&Ok(true)));
17996        assert!(!drained_after_quiescence_wait(&Ok(false)));
17997        assert!(!drained_after_quiescence_wait(&Err(
17998            SuperviseError::StatePoisoned { module_id: None }
17999        )));
18000    }
18001
18002    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
18003    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
18004    /// already reaped out-of-band) still leaves a terminal record rather than none
18005    /// at all. Triggering the real `wait()` I/O error from an integration test would
18006    /// need a genuine already-reaped-child race, which is OS-specific and not
18007    /// something this suite attempts elsewhere; this test instead verifies the
18008    /// record produced for that arm end-to-end through the real `TerminalRing`, and
18009    /// the call site itself is verified by inspection to sit in that exact arm.
18010    #[test]
18011    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
18012        let ring = Arc::new(Mutex::new(TerminalRing::new(
18013            TerminalRingConfig::default(),
18014            0,
18015        )));
18016        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
18017
18018        let snapshot = ring.lock().unwrap().snapshot();
18019        assert_eq!(snapshot.entries.len(), 1);
18020        let entry = &snapshot.entries[0];
18021        assert_eq!(entry.exit_code, None);
18022        assert_eq!(entry.exit_signal, None);
18023        assert_eq!(entry.disposition, TerminalDisposition::Failed);
18024    }
18025
18026    /// A launch is numbered before it starts. One that never runs keeps its
18027    /// number, and the next launch of the module is numbered after it.
18028    #[test]
18029    fn spawn_generation_of_a_failed_launch_is_never_reused() {
18030        let feed = super::SpawnEventFeed::default();
18031        feed.configure_incarnation("launch-numbering".to_string());
18032        let (incarnation, failed) = feed.allocate_generation("relaunched");
18033        assert_eq!(incarnation, "launch-numbering");
18034        let (_, next) = feed.allocate_generation("relaunched");
18035        assert!(next > failed);
18036        assert_eq!(feed.emit_spawned("relaunched", Some(next), 7, 1), next);
18037        // A process with no pre-allocated number is numbered after both.
18038        assert!(feed.emit_spawned("relaunched", None, 8, 2) > next);
18039        assert_eq!(
18040            feed.allocate_generation("other").1,
18041            1,
18042            "numbering is per module"
18043        );
18044    }
18045
18046    #[test]
18047    fn wait_error_exit_path_preserves_spawn_event_density() {
18048        let feed = super::SpawnEventFeed::default();
18049        feed.configure_incarnation("wait-error-density".to_string());
18050        feed.emit_spawned("wait-error", None, 41, 1);
18051        let ring = Arc::new(Mutex::new(TerminalRing::new(
18052            TerminalRingConfig::default(),
18053            0,
18054        )));
18055
18056        record_wait_error_terminal("wait-error", &ring, &feed);
18057        feed.emit_spawned("after-wait-error", None, 42, 2);
18058
18059        let state = feed.0.lock().unwrap();
18060        let sequences = state
18061            .events
18062            .iter()
18063            .map(|event| event.cursor.seq)
18064            .collect::<Vec<_>>();
18065        assert_eq!(sequences, vec![1, 2, 3]);
18066        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
18067        assert_eq!(state.events[1].exit_code, None);
18068        assert_eq!(state.events[1].exit_signal, None);
18069    }
18070
18071    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
18072    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
18073    /// not a clean exit it never actually observed.
18074    #[test]
18075    fn wait_error_exit_report_is_classified_as_a_crash() {
18076        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
18077    }
18078}
18079
18080#[cfg(all(test, unix))]
18081mod restart_ordering_tests {
18082    use super::*;
18083    use std::io::Write;
18084    use tokio::io::AsyncReadExt;
18085
18086    #[test]
18087    #[ignore = "child process fixture"]
18088    fn serving_child() {
18089        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
18090            return;
18091        };
18092        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
18093        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
18094        for stream in listener.incoming() {
18095            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
18096        }
18097    }
18098
18099    async fn restart_before_publication(requests: usize) {
18100        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
18101        let address_file = home.path().join("address");
18102        let spec = ModuleSpec {
18103            script: None,
18104            module_id: "restart-serving-child".into(),
18105            program: std::env::current_exe().unwrap(),
18106            args: vec![
18107                "--exact".into(),
18108                "supervise::restart_ordering_tests::serving_child".into(),
18109                "--ignored".into(),
18110                "--nocapture".into(),
18111            ],
18112            env: vec![(
18113                "RESTART_TEST_ADDRESS_FILE".into(),
18114                address_file.display().to_string(),
18115            )],
18116            reserved: false,
18117            reserved_prefixes: vec![],
18118            protocol: ModuleProtocol::None,
18119            overlap: Default::default(),
18120            carry_containment: false,
18121        };
18122        let mut supervisor =
18123            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18124        let (spawned_tx, spawned_rx) = oneshot::channel();
18125        let spawned_tx = Mutex::new(Some(spawned_tx));
18126        let (release_tx, release_rx) = std::sync::mpsc::channel();
18127        let release_rx = Mutex::new(release_rx);
18128        // Hold publication, not the child: it must answer a real request before
18129        // the restart enters the queue, even though status still has generation 0.
18130        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
18131            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
18132            release_rx
18133                .lock()
18134                .unwrap()
18135                .recv_timeout(Duration::from_secs(20))
18136                .unwrap();
18137        })));
18138        let registered = supervisor
18139            .register_configured_with_health(
18140                spec.clone(),
18141                true,
18142                HealthConfig::default(),
18143                None,
18144                RestartPolicy::new(3, Duration::ZERO),
18145                DEFAULT_REGISTER_TIMEOUT,
18146            )
18147            .unwrap();
18148        let module = registered.module.clone();
18149        let startup = tokio::task::spawn_blocking(move || {
18150            supervisor.start_registered_configured(registered, None, true)
18151        });
18152        spawned_rx.await.unwrap();
18153        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18154            let address = loop {
18155                if let Ok(address) = std::fs::read_to_string(&address_file) {
18156                    if !address.is_empty() {
18157                        break address;
18158                    }
18159                }
18160                tokio::time::sleep(Duration::from_millis(1)).await;
18161            };
18162            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
18163            let mut reply = String::new();
18164            stream.read_to_string(&mut reply).await.unwrap();
18165            reply.trim().parse::<u32>().unwrap()
18166        })
18167        .await
18168        .expect("child must serve before generation publication");
18169        assert_eq!(
18170            lock_snapshot(&module.inner.snapshot)
18171                .unwrap()
18172                .spawn_generation,
18173            0
18174        );
18175        let mut replies = Vec::new();
18176        for _ in 0..requests {
18177            let mut restart = Box::pin(module.restart(Some(0)));
18178            // Poll the public API through queue admission, leaving only its ACK
18179            // pending. The monitor cannot consume the command until released.
18180            std::future::poll_fn(|cx| {
18181                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
18182                std::task::Poll::Ready(())
18183            })
18184            .await;
18185            replies.push(restart);
18186        }
18187        release_tx.send(()).unwrap();
18188        startup.await.unwrap().unwrap();
18189        for reply in replies {
18190            reply.await.unwrap();
18191        }
18192        // Restart schedules its replacement after completing the handler. Wait
18193        // for that separate actor turn before placing the command barrier.
18194        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
18195            loop {
18196                if lock_snapshot(&module.inner.snapshot)
18197                    .unwrap()
18198                    .spawn_generation
18199                    > 1
18200                {
18201                    break;
18202                }
18203                tokio::time::sleep(Duration::from_millis(1)).await;
18204            }
18205        })
18206        .await;
18207        // Commands are handled serially. The update-spec reply proves every
18208        // restart handler has finished before we inspect the snapshot; restart's
18209        // own ACK only acknowledges initiation, not replacement completion.
18210        module.update_spec_for_test(spec).await.unwrap();
18211        let (generation, pid) = {
18212            let state = lock_snapshot(&module.inner.snapshot).unwrap();
18213            (state.spawn_generation, state.pid)
18214        };
18215        module.stop().await.unwrap();
18216        replacement.expect("restart must publish a replacement for the already-serving child");
18217        assert_ne!(
18218            pid,
18219            Some(old_pid),
18220            "a serving child predating the restart must be replaced"
18221        );
18222        assert_eq!(generation, 2, "queued restarts must share one replacement");
18223    }
18224
18225    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18226    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
18227        restart_before_publication(1).await;
18228    }
18229
18230    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18231    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
18232        restart_before_publication(2).await;
18233    }
18234
18235    #[tokio::test]
18236    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
18237        let supervisor =
18238            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18239        let registered = supervisor
18240            .register_configured_with_health(
18241                ModuleSpec {
18242                    script: None,
18243                    module_id: "queued-before-replacement".into(),
18244                    // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on
18245                    // macOS a module whose running image is not the configured
18246                    // program is refused once its exec is confirmed (see
18247                    // `record_write_stall_tests::sleeping_spec`).
18248                    program: "/bin/sleep".into(),
18249                    args: vec!["600".into()],
18250                    env: vec![],
18251                    reserved: false,
18252                    reserved_prefixes: vec![],
18253                    protocol: ModuleProtocol::None,
18254                    overlap: Default::default(),
18255                    carry_containment: false,
18256                },
18257                true,
18258                HealthConfig::default(),
18259                None,
18260                RestartPolicy::default(),
18261                DEFAULT_REGISTER_TIMEOUT,
18262            )
18263            .unwrap();
18264        let RegisteredConfiguredModule {
18265            mut spec,
18266            mut runtime,
18267            module,
18268            mut commands,
18269        } = registered;
18270        let snapshot = &module.inner.snapshot;
18271        let mut restart = Box::pin(module.restart(Some(0)));
18272        std::future::poll_fn(|cx| {
18273            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
18274            std::task::Poll::Ready(())
18275        })
18276        .await;
18277        // Drive the actor explicitly so a replacement is definitely spawned
18278        // after receipt, but before this queued command is dequeued.
18279        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
18280        let replacement_pid = child.as_ref().unwrap().id();
18281        let command = commands.recv().await.unwrap();
18282        assert!(
18283            handle_supervisor_command(
18284                command,
18285                &mut spec,
18286                &mut runtime,
18287                &supervisor.registry,
18288                &supervisor.process_liveness,
18289                snapshot,
18290                &mut child,
18291                &mut commands,
18292                &mut VecDeque::new(),
18293            )
18294            .await
18295        );
18296        restart.await.unwrap();
18297        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
18298        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
18299        if let Some(mut child) = child {
18300            child.start_kill().unwrap();
18301            child.wait().await.unwrap();
18302        }
18303        assert_eq!(
18304            kept_pid, replacement_pid,
18305            "a newer replacement must survive"
18306        );
18307        assert!(
18308            !pending,
18309            "a satisfied restart must not schedule another spawn"
18310        );
18311    }
18312}
18313
18314#[cfg(all(test, unix))]
18315mod health_event_tests {
18316    use super::*;
18317    use cortexkit_test_support::ScratchDir;
18318    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
18319
18320    struct Actor {
18321        module: SupervisedModule,
18322        registry: Arc<Registry>,
18323        forwarding: Arc<ForwardingTable>,
18324        spec: ModuleSpec,
18325        health: HealthConfig,
18326        rx: mpsc::Receiver<crate::router::OutboundFrame>,
18327        roster: ChildRoster,
18328        _home: ScratchDir,
18329    }
18330
18331    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
18332        // OS exec and socket readiness are real I/O. Stay runnable while waiting
18333        // for an observed condition, rather than letting paused time auto-advance
18334        // deadlines or assuming a fixed number of yields finishes the work.
18335        let deadline = std::time::Instant::now() + Duration::from_secs(10);
18336        let tick = Instant::now();
18337        loop {
18338            if let Some(value) = observe() {
18339                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
18340                return value;
18341            }
18342            assert!(
18343                std::time::Instant::now() < deadline,
18344                "timed out waiting for {reason}"
18345            );
18346            tokio::task::yield_now().await;
18347        }
18348    }
18349
18350    impl Actor {
18351        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
18352            Self::start_with_policy(
18353                protocol,
18354                cadence,
18355                DEFAULT_REGISTER_TIMEOUT,
18356                RestartPolicy::new(3, Duration::ZERO),
18357            )
18358            .await
18359        }
18360
18361        async fn start_with_policy(
18362            protocol: ModuleProtocol,
18363            cadence: Duration,
18364            register_timeout: Duration,
18365            policy: RestartPolicy,
18366        ) -> Self {
18367            let home = ScratchDir::new("health-events");
18368            let registry = Arc::new(Registry::default());
18369            let forwarding = Arc::new(ForwardingTable::default());
18370            let health = HealthConfig {
18371                cadence,
18372                deadline: Duration::from_secs(3600),
18373                ..HealthConfig::default()
18374            };
18375            let supervisor = Supervisor::new_for_test(registry.clone(), policy)
18376                .with_forwarding(forwarding.clone())
18377                .with_health_config(health.clone());
18378            let spec = ModuleSpec {
18379                script: None,
18380                module_id: "event-health-child".into(),
18381                // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on macOS
18382                // the privacy trampoline refuses a module whose running image
18383                // is not the configured program, and `start` below waits for
18384                // the process to be confirmed (`reported_pid`). The shell form
18385                // passed only while the image check ran before the shell had
18386                // replaced itself with `sleep` (see
18387                // `record_write_stall_tests::sleeping_spec`).
18388                program: PathBuf::from("/bin/sleep"),
18389                // The process stays alive but has no real bus peer. Registrations
18390                // below exercise the same registry writes as accepted HELLOs.
18391                args: vec!["600".into()],
18392                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18393                    .into_iter()
18394                    .map(|key| (key.into(), home.path().display().to_string()))
18395                    .collect(),
18396                reserved: false,
18397                reserved_prefixes: vec![],
18398                protocol,
18399                overlap: Default::default(),
18400                carry_containment: false,
18401            };
18402            let module = supervisor
18403                .supervise_configured_requested(
18404                    spec.clone(),
18405                    true,
18406                    health.clone(),
18407                    Some(0),
18408                    policy,
18409                    register_timeout,
18410                    None,
18411                )
18412                .unwrap();
18413            let (_, rx) = mpsc::channel(8);
18414            let actor = Self {
18415                module,
18416                registry,
18417                forwarding,
18418                spec,
18419                health,
18420                rx,
18421                roster: supervisor.child_roster.clone(),
18422                _home: home,
18423            };
18424            actor
18425                .wait_for_select(|checkpoint| checkpoint.generation > 0)
18426                .await;
18427            actor
18428        }
18429
18430        async fn expire_registration_wait(&self, remaining: Duration) {
18431            tokio::time::advance(remaining).await;
18432            wait_for("registration timeout TERM grace", || {
18433                lock_snapshot(&self.module.inner.snapshot)
18434                    .unwrap()
18435                    .actor_select
18436                    .is_none()
18437                    .then_some(())
18438            })
18439            .await;
18440            tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
18441        }
18442
18443        fn turns(&self) -> u64 {
18444            lock_snapshot(&self.module.inner.snapshot)
18445                .unwrap()
18446                .actor_turns
18447        }
18448
18449        async fn wait_for_select(
18450            &self,
18451            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
18452        ) -> ActorSelectCheckpoint {
18453            wait_for("actor select checkpoint", || {
18454                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
18455                state.actor_select.filter(|checkpoint| {
18456                    checkpoint.generation == state.spawn_generation
18457                        && state.state == ModuleState::Running
18458                        && state.process_alive
18459                        && state.reported_pid().is_some()
18460                        && expected(checkpoint)
18461                })
18462            })
18463            .await
18464        }
18465
18466        fn hello(&mut self, connection: u64, health: bool) {
18467            let manifest =
18468                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
18469                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
18470                    .build();
18471            let connection = ConnectionId::new(connection);
18472            let (tx, rx) = mpsc::channel(8);
18473            self.rx = rx;
18474            self.forwarding
18475                .register_module_connection(
18476                    connection,
18477                    self.spec.module_id.clone(),
18478                    subc_protocol::PROTOCOL_VERSION,
18479                    Concurrency::ModuleManaged,
18480                    FrameSink::new(tx),
18481                )
18482                .unwrap();
18483            self.registry
18484                .register_with_control_ops(
18485                    manifest,
18486                    subc_protocol::PROTOCOL_VERSION,
18487                    connection,
18488                    if health {
18489                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
18490                    } else {
18491                        vec![]
18492                    },
18493                )
18494                .unwrap();
18495        }
18496
18497        async fn probe(&mut self) -> crate::router::OutboundFrame {
18498            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
18499                Ok(frame) => Some(frame),
18500                Err(mpsc::error::TryRecvError::Empty) => None,
18501                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
18502            })
18503            .await;
18504            let body: Value = serde_json::from_slice(&frame.body).unwrap();
18505            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
18506            frame
18507        }
18508
18509        fn no_probe(&mut self) {
18510            assert!(matches!(
18511                self.rx.try_recv(),
18512                Err(mpsc::error::TryRecvError::Empty)
18513            ));
18514        }
18515
18516        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
18517            self.forwarding
18518                .complete_module_control_rpc(
18519                    ConnectionId::new(connection),
18520                    frame.header.corr,
18521                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
18522                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
18523                        status: HealthStatus::Ok,
18524                        detail: None,
18525                        metrics: None,
18526                    }),
18527                )
18528                .unwrap();
18529        }
18530    }
18531
18532    #[tokio::test(start_paused = true)]
18533    async fn registration_timeout_kills_at_120_seconds_and_respawns() {
18534        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18535        let initial = actor.module.status().unwrap();
18536        tokio::time::advance(Duration::from_secs(119)).await;
18537        tokio::task::yield_now().await;
18538        assert_eq!(actor.module.status().unwrap().pid, initial.pid);
18539        assert!(actor.module.terminal_history().entries.is_empty());
18540        actor.expire_registration_wait(Duration::from_secs(1)).await;
18541        actor
18542            .wait_for_select(|at| at.generation > initial.spawn_generation)
18543            .await;
18544        let status = actor.module.status().unwrap();
18545        assert_ne!(status.pid, initial.pid);
18546        assert!(!cortexkit_test_support::process_alive(
18547            initial.pid.unwrap() as i32
18548        ));
18549        assert_eq!(status.restart_count, 1);
18550        assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
18551        let entries = actor.module.terminal_history().entries;
18552        assert_eq!(entries.len(), 1);
18553        assert_eq!(
18554            entries[0].disposition_detail.as_deref(),
18555            Some("registration_timeout")
18556        );
18557        actor.module.stop().await.unwrap();
18558    }
18559
18560    #[tokio::test(start_paused = true)]
18561    async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
18562        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18563        let pid = actor.module.status().unwrap().pid;
18564        tokio::time::advance(Duration::from_secs(119)).await;
18565        actor.hello(1, false);
18566        let registered = actor
18567            .wait_for_select(|at| at.registered_connection.is_some())
18568            .await;
18569        assert_eq!(
18570            registered.wake_after, None,
18571            "HELLO must cancel the deadline"
18572        );
18573        tokio::time::advance(Duration::from_secs(300)).await;
18574        tokio::task::yield_now().await;
18575        assert_eq!(actor.module.status().unwrap().pid, pid);
18576        assert!(actor.module.terminal_history().entries.is_empty());
18577        // A disconnect after the first HELLO must not re-arm a startup timer.
18578        actor
18579            .registry
18580            .deregister_connection(ConnectionId::new(1))
18581            .unwrap();
18582        actor
18583            .wait_for_select(|at| at.registered_connection.is_none())
18584            .await;
18585        tokio::time::advance(Duration::from_secs(300)).await;
18586        tokio::task::yield_now().await;
18587        assert_eq!(actor.module.status().unwrap().pid, pid);
18588        assert!(actor.module.terminal_history().entries.is_empty());
18589        actor.module.stop().await.unwrap();
18590
18591        let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18592        let pid = transient.module.status().unwrap().pid;
18593        tokio::time::advance(Duration::from_secs(119)).await;
18594        let turns = transient.turns();
18595        // Both writes happen before the supervisor can refresh the registry.
18596        // An accepted HELLO must remain evidence after its connection is gone.
18597        transient.hello(1, false);
18598        transient
18599            .registry
18600            .deregister_connection(ConnectionId::new(1))
18601            .unwrap();
18602        let parked = transient.wait_for_select(|at| at.turn > turns).await;
18603        assert_eq!(
18604            parked.wake_after, None,
18605            "transient HELLO must cancel the deadline"
18606        );
18607        tokio::time::advance(Duration::from_secs(300)).await;
18608        tokio::task::yield_now().await;
18609        assert_eq!(transient.module.status().unwrap().pid, pid);
18610        assert!(transient.module.terminal_history().entries.is_empty());
18611        transient.module.stop().await.unwrap();
18612    }
18613
18614    #[tokio::test(start_paused = true)]
18615    async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
18616        let actor = Actor::start_with_policy(
18617            ModuleProtocol::Subc,
18618            Duration::from_secs(30),
18619            Duration::from_secs(10),
18620            RestartPolicy::new(1, Duration::ZERO),
18621        )
18622        .await;
18623        let generation = actor.module.status().unwrap().spawn_generation;
18624        actor
18625            .expire_registration_wait(Duration::from_secs(10))
18626            .await;
18627        actor.wait_for_select(|at| at.generation > generation).await;
18628        actor
18629            .expire_registration_wait(Duration::from_secs(10))
18630            .await;
18631        let retry = wait_for("failed retry schedule", || {
18632            let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
18633            (state.state == ModuleState::Failed)
18634                .then_some(state.failed_retry.pending)
18635                .flatten()
18636        })
18637        .await;
18638        assert_eq!(retry.delay, Duration::from_secs(600));
18639        assert_eq!(retry.attempt, 1);
18640        let entries = actor.module.terminal_history().entries;
18641        assert_eq!(entries.len(), 2);
18642        assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
18643        assert!(entries[1]
18644            .disposition_detail
18645            .as_deref()
18646            .unwrap()
18647            .contains("registration_timeout"));
18648        tokio::time::advance(Duration::from_secs(599)).await;
18649        tokio::task::yield_now().await;
18650        assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
18651        tokio::time::advance(Duration::from_secs(1)).await;
18652        actor
18653            .wait_for_select(|at| at.generation > generation + 1)
18654            .await;
18655        // The slow retry is another ordinary spawn, with its own HELLO budget.
18656        actor
18657            .expire_registration_wait(Duration::from_secs(10))
18658            .await;
18659        wait_for("slow retry registration timeout", || {
18660            (actor.module.terminal_history().entries.len() == 3).then_some(())
18661        })
18662        .await;
18663        actor.module.stop().await.unwrap();
18664    }
18665
18666    #[tokio::test(start_paused = true)]
18667    async fn registration_timeout_honours_ten_second_override() {
18668        let actor = Actor::start_with_policy(
18669            ModuleProtocol::Subc,
18670            Duration::from_secs(30),
18671            Duration::from_secs(10),
18672            RestartPolicy::new(3, Duration::ZERO),
18673        )
18674        .await;
18675        let generation = actor.module.status().unwrap().spawn_generation;
18676        tokio::time::advance(Duration::from_secs(9)).await;
18677        tokio::task::yield_now().await;
18678        assert!(actor.module.terminal_history().entries.is_empty());
18679        actor.expire_registration_wait(Duration::from_secs(1)).await;
18680        actor.wait_for_select(|at| at.generation > generation).await;
18681        assert_eq!(actor.module.terminal_history().entries.len(), 1);
18682        actor.module.stop().await.unwrap();
18683    }
18684
18685    #[tokio::test]
18686    async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
18687        use std::os::unix::process::{CommandExt, ExitStatusExt};
18688
18689        struct Helper(std::process::Child);
18690        impl Drop for Helper {
18691            fn drop(&mut self) {
18692                // This handle still owns the helper until wait returns. Never
18693                // clean up its old process group after the leader is reaped.
18694                if matches!(self.0.try_wait(), Ok(None)) {
18695                    let _ = self.0.kill();
18696                    let _ = self.0.wait();
18697                }
18698            }
18699        }
18700
18701        let home = ScratchDir::new("registration-timeout-helper");
18702        let supervisor = Supervisor::new_for_test(
18703            Arc::new(Registry::default()),
18704            RestartPolicy::new(0, Duration::ZERO),
18705        );
18706        let spec = ModuleSpec {
18707            script: None,
18708            module_id: "registration-timeout-group".into(),
18709            program: "/bin/sleep".into(),
18710            args: vec!["120".into()],
18711            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18712                .into_iter()
18713                .map(|key| (key.into(), home.path().display().to_string()))
18714                .collect(),
18715            reserved: false,
18716            reserved_prefixes: vec![],
18717            protocol: ModuleProtocol::Subc,
18718            overlap: Default::default(),
18719            carry_containment: false,
18720        };
18721        let module = supervisor
18722            .supervise_configured_requested(
18723                spec,
18724                true,
18725                HealthConfig::default(),
18726                None,
18727                RestartPolicy::new(0, Duration::ZERO),
18728                Duration::from_secs(10),
18729                None,
18730            )
18731            .unwrap();
18732        let leader = timeout(Duration::from_secs(5), async {
18733            loop {
18734                if let Some(pid) = module.status().unwrap().pid {
18735                    break pid;
18736                }
18737                sleep(Duration::from_millis(5)).await;
18738            }
18739        })
18740        .await
18741        .expect("supervised group leader must be reported before the deadline");
18742        assert!(leader > 1);
18743        let ready = home.join("helper.ready");
18744        // Own the helper separately so even a broken group kill can be cleaned
18745        // up by its own unreaped child handle, without signalling a stale pgid.
18746        let mut helper = Helper(
18747            std::process::Command::new("/bin/sh")
18748                .args([
18749                    "-c",
18750                    "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
18751                    "helper",
18752                ])
18753                .arg(&ready)
18754                .process_group(i32::try_from(leader).unwrap())
18755                .stdout(std::process::Stdio::null())
18756                .stderr(std::process::Stdio::null())
18757                .spawn()
18758                .unwrap(),
18759        );
18760        timeout(Duration::from_secs(5), async {
18761            while !ready.exists() {
18762                assert!(helper.0.try_wait().unwrap().is_none());
18763                sleep(Duration::from_millis(5)).await;
18764            }
18765        })
18766        .await
18767        .expect("helper must install TERM ignore before the registration deadline");
18768        let helper_pid =
18769            rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
18770        rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
18771        sleep(Duration::from_millis(20)).await;
18772        assert!(
18773            helper.0.try_wait().unwrap().is_none(),
18774            "helper must ignore TERM"
18775        );
18776
18777        timeout(Duration::from_secs(20), async {
18778            while module.state().unwrap() != ModuleState::Failed {
18779                sleep(Duration::from_millis(5)).await;
18780            }
18781        })
18782        .await
18783        .expect("registration timeout must stop and reap the group leader");
18784        let helper_exit = timeout(Duration::from_secs(3), async {
18785            loop {
18786                if let Some(status) = helper.0.try_wait().unwrap() {
18787                    break status;
18788                }
18789                sleep(Duration::from_millis(5)).await;
18790            }
18791        })
18792        .await
18793        .expect("TERM-ignoring helper survived the registration timeout's group KILL");
18794        assert_eq!(helper_exit.signal(), Some(9));
18795        assert!(!cortexkit_test_support::process_alive(
18796            i32::try_from(leader).unwrap()
18797        ));
18798        assert_eq!(
18799            module.status().unwrap().last_exit.unwrap().kind,
18800            ExitKind::Crash
18801        );
18802        assert!(module.terminal_history().entries[0]
18803            .disposition_detail
18804            .as_deref()
18805            .unwrap()
18806            .contains("registration_timeout"));
18807        module.stop().await.unwrap();
18808    }
18809
18810    #[tokio::test(start_paused = true)]
18811    async fn registration_timeout_does_not_count_an_earlier_exit() {
18812        let actor = Actor::start_with_policy(
18813            ModuleProtocol::Subc,
18814            Duration::from_secs(30),
18815            DEFAULT_REGISTER_TIMEOUT,
18816            RestartPolicy::new(0, Duration::ZERO),
18817        )
18818        .await;
18819        let pid = actor.module.status().unwrap().pid.unwrap();
18820        rustix::process::kill_process(
18821            rustix::process::Pid::from_raw(pid as i32).unwrap(),
18822            rustix::process::Signal::TERM,
18823        )
18824        .unwrap();
18825        wait_for("early exit", || {
18826            (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
18827        })
18828        .await;
18829        tokio::time::advance(Duration::from_secs(121)).await;
18830        tokio::task::yield_now().await;
18831        let entries = actor.module.terminal_history().entries;
18832        assert_eq!(entries.len(), 1);
18833        assert!(!entries[0]
18834            .disposition_detail
18835            .as_deref()
18836            .unwrap()
18837            .contains("registration_timeout"));
18838        assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
18839        actor.module.stop().await.unwrap();
18840    }
18841
18842    #[tokio::test(start_paused = true)]
18843    async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
18844        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18845        let pid = actor.module.status().unwrap().pid;
18846        tokio::time::advance(Duration::from_secs(119)).await;
18847        actor.roster.close();
18848        let turns = actor.turns();
18849        tokio::time::advance(Duration::from_secs(300)).await;
18850        let parked = actor.wait_for_select(|at| at.turn > turns).await;
18851        assert_eq!(parked.wake_after, None);
18852        assert_eq!(actor.module.status().unwrap().pid, pid);
18853        assert!(actor.module.terminal_history().entries.is_empty());
18854        actor.module.stop().await.unwrap();
18855        let status = actor.module.status().unwrap();
18856        assert_eq!(status.restart_count, 0);
18857        assert_eq!(
18858            actor.module.terminal_history().entries[0].disposition,
18859            TerminalDisposition::DaemonShutdown
18860        );
18861    }
18862
18863    #[tokio::test(start_paused = true)]
18864    async fn no_health_children_park_for_sixty_seconds() {
18865        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
18866        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18867        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18868        unadvertised.hello(1, false);
18869        // Include any one-off registration wake in the budget; receiving a
18870        // no-health HELLO must not turn a parked child into a polling child.
18871        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
18872        // a single 60 s jump would only observe one expired timer.
18873        for _ in 0..6000 {
18874            tokio::time::advance(Duration::from_millis(10)).await;
18875            tokio::task::yield_now().await;
18876        }
18877        for actor in [&none, &unregistered, &unadvertised] {
18878            assert!(
18879                actor.turns() <= 5,
18880                "no-health actor woke {} times",
18881                actor.turns()
18882            );
18883            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
18884        }
18885    }
18886
18887    #[tokio::test(start_paused = true)]
18888    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
18889        // With a zero cadence the first probe is due at once, so this measures
18890        // only how fast the registration notification starts health probing,
18891        // not the normal 30-33 s wait before a first probe (checked below).
18892        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
18893        let tick = Instant::now();
18894        actor.hello(1, true);
18895        actor.probe().await;
18896        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
18897    }
18898
18899    #[tokio::test(start_paused = true)]
18900    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
18901        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18902        let hello_tick = Instant::now();
18903        actor.hello(1, true);
18904        let armed = actor
18905            .wait_for_select(|checkpoint| {
18906                checkpoint.registered_connection == Some(ConnectionId::new(1))
18907                    && checkpoint.next_probe_at.is_some()
18908            })
18909            .await;
18910        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18911        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
18912        let first_deadline = hello_tick + delay;
18913        assert_eq!(armed.next_probe_at, Some(first_deadline));
18914        tokio::time::advance(Duration::from_secs(29)).await;
18915        let turns = actor.turns();
18916        actor
18917            .registry
18918            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
18919            .unwrap();
18920        let updated = actor
18921            .wait_for_select(|checkpoint| checkpoint.turn > turns)
18922            .await;
18923        assert_eq!(
18924            updated.next_probe_at,
18925            Some(first_deadline),
18926            "catalog update must not reset cadence"
18927        );
18928        actor.no_probe();
18929        tokio::time::advance(first_deadline - Instant::now()).await;
18930        let frame = actor.probe().await;
18931        actor.answer(1, frame);
18932        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
18933        let next_deadline = Instant::now() + next;
18934        let rearmed = actor
18935            .wait_for_select(|checkpoint| {
18936                checkpoint
18937                    .next_probe_at
18938                    .is_some_and(|deadline| deadline > first_deadline)
18939            })
18940            .await;
18941        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
18942        tokio::time::advance(next - Duration::from_nanos(1)).await;
18943        actor.no_probe();
18944        tokio::time::advance(Duration::from_nanos(1)).await;
18945        actor.probe().await;
18946    }
18947
18948    #[tokio::test(start_paused = true)]
18949    async fn disconnect_disarms_and_reconnect_rearms_health() {
18950        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18951        actor.hello(1, true);
18952        actor
18953            .wait_for_select(|checkpoint| {
18954                checkpoint.registered_connection == Some(ConnectionId::new(1))
18955                    && checkpoint.next_probe_at.is_some()
18956            })
18957            .await;
18958        let turns = actor.turns();
18959        let tick = Instant::now();
18960        actor
18961            .registry
18962            .deregister_connection(ConnectionId::new(1))
18963            .unwrap();
18964        let disarmed = actor
18965            .wait_for_select(|checkpoint| {
18966                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
18967            })
18968            .await;
18969        assert_eq!(disarmed.next_probe_at, None);
18970        assert_eq!(disarmed.wake_after, None);
18971        assert_eq!(
18972            Instant::now(),
18973            tick,
18974            "disconnect must disarm in the notification tick"
18975        );
18976        tokio::time::advance(Duration::from_secs(60)).await;
18977        actor.no_probe();
18978        let reconnect_tick = Instant::now();
18979        actor.hello(2, true);
18980        let rearmed = actor
18981            .wait_for_select(|checkpoint| {
18982                checkpoint.registered_connection == Some(ConnectionId::new(2))
18983                    && checkpoint.next_probe_at.is_some()
18984            })
18985            .await;
18986        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18987        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
18988        tokio::time::advance(delay).await;
18989        actor.probe().await;
18990    }
18991
18992    #[tokio::test(start_paused = true)]
18993    async fn reregistration_without_health_disarms_the_old_schedule() {
18994        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18995        actor.hello(1, true);
18996        actor
18997            .wait_for_select(|checkpoint| {
18998                checkpoint.registered_connection == Some(ConnectionId::new(1))
18999                    && checkpoint.next_probe_at.is_some()
19000            })
19001            .await;
19002        actor
19003            .registry
19004            .deregister_connection(ConnectionId::new(1))
19005            .unwrap();
19006        actor.hello(2, false);
19007        let disarmed = actor
19008            .wait_for_select(|checkpoint| {
19009                checkpoint.registered_connection == Some(ConnectionId::new(2))
19010            })
19011            .await;
19012        assert_eq!(disarmed.next_probe_at, None);
19013        assert_eq!(disarmed.wake_after, None);
19014        tokio::time::advance(Duration::from_secs(60)).await;
19015        actor.no_probe();
19016    }
19017
19018    #[tokio::test(start_paused = true)]
19019    async fn restart_waits_for_its_own_hello_before_rearming_health() {
19020        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
19021        actor.hello(1, true);
19022        let armed = actor
19023            .wait_for_select(|checkpoint| {
19024                checkpoint.registered_connection == Some(ConnectionId::new(1))
19025                    && checkpoint.next_probe_at.is_some()
19026            })
19027            .await;
19028        // Simulate the old process's connection closing before the restart
19029        // tears the process down. The replacement process has started but has
19030        // not sent its HELLO, so it is not registered.
19031        actor
19032            .registry
19033            .deregister_connection(ConnectionId::new(1))
19034            .unwrap();
19035        actor.module.restart(Some(0)).await.unwrap();
19036        let parked = actor
19037            .wait_for_select(|checkpoint| {
19038                checkpoint.generation > armed.generation
19039                    && checkpoint.registered_connection.is_none()
19040                    && checkpoint.next_probe_at.is_none()
19041            })
19042            .await;
19043        assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
19044        actor.rx.close();
19045        // Drain the stop notice the restart queued for the old process's connection,
19046        // so it isn't mistaken for traffic to the replacement.
19047        while actor.rx.try_recv().is_ok() {}
19048        let turns = actor.turns();
19049        tokio::time::advance(Duration::from_secs(60)).await;
19050        let after = actor.turns();
19051        eprintln!(
19052            "restart park turns: before={turns} after={after} delta={}",
19053            after - turns
19054        );
19055        assert_eq!(after, turns, "replacement must park before HELLO");
19056        let hello_tick = Instant::now();
19057        actor.hello(2, true);
19058        let rearmed = actor
19059            .wait_for_select(|checkpoint| {
19060                checkpoint.registered_connection == Some(ConnectionId::new(2))
19061                    && checkpoint.next_probe_at.is_some()
19062            })
19063            .await;
19064        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
19065        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
19066        tokio::time::advance(delay).await;
19067        actor.probe().await;
19068    }
19069
19070    #[tokio::test(start_paused = true)]
19071    async fn http_probe_waits_a_full_cadence_for_each_spawn() {
19072        use tokio::{
19073            io::{AsyncReadExt, AsyncWriteExt},
19074            net::TcpListener,
19075        };
19076        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
19077        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
19078        actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
19079        let (started_tx, mut started_rx) = mpsc::channel(8);
19080        let peer = tokio::spawn(async move {
19081            loop {
19082                let (mut socket, _) = listener.accept().await.unwrap();
19083                let mut request = [0; 1024];
19084                assert!(socket.read(&mut request).await.unwrap() > 0);
19085                started_tx.send(Instant::now()).await.unwrap();
19086                socket
19087                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
19088                    .await
19089                    .unwrap();
19090            }
19091        });
19092        actor
19093            .module
19094            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19095            .await
19096            .unwrap();
19097        let first_tick = Instant::now();
19098        let first = actor
19099            .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
19100            .await;
19101        assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
19102        assert!(started_rx.try_recv().is_err());
19103
19104        // Restart just before the old deadline: reusing it would give the new
19105        // process only one second to start, rather than a full cadence.
19106        tokio::time::advance(
19107            first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
19108        )
19109        .await;
19110        actor.module.restart(Some(0)).await.unwrap();
19111        let spawn_tick = Instant::now();
19112        let replacement = actor
19113            .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
19114            .await;
19115        let deadline = replacement.next_probe_at.unwrap();
19116        assert!(
19117            deadline >= spawn_tick + actor.health.cadence,
19118            "a replacement must not inherit its predecessor's probe deadline"
19119        );
19120        tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
19121        assert!(
19122            started_rx.try_recv().is_err(),
19123            "no HTTP probe during startup grace"
19124        );
19125        tokio::time::advance(deadline - Instant::now()).await;
19126        let landed = wait_for("first replacement HTTP probe", || {
19127            started_rx.try_recv().ok()
19128        })
19129        .await;
19130        assert!(landed >= spawn_tick + actor.health.cadence);
19131        wait_for("healthy replacement", || {
19132            (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
19133                .then_some(())
19134        })
19135        .await;
19136        peer.abort();
19137    }
19138
19139    #[tokio::test(start_paused = true)]
19140    async fn rescan_adds_and_removes_http_health_without_polling() {
19141        use tokio::{
19142            io::{AsyncReadExt, AsyncWriteExt},
19143            net::TcpListener,
19144        };
19145        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
19146        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
19147        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
19148        // One millisecond separates successive probes without a polling delay.
19149        actor.health.cadence = Duration::from_millis(1);
19150        let (started_tx, mut started_rx) = mpsc::channel(8);
19151        let peer = tokio::spawn(async move {
19152            loop {
19153                let (mut socket, _) = listener.accept().await.unwrap();
19154                let mut request = [0; 1024];
19155                assert!(socket.read(&mut request).await.unwrap() > 0);
19156                started_tx.send(()).await.unwrap();
19157                socket
19158                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
19159                    .await
19160                    .unwrap();
19161            }
19162        });
19163        let tick = Instant::now();
19164        let turns = actor.turns();
19165        actor
19166            .module
19167            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19168            .await
19169            .unwrap();
19170        let armed = actor
19171            .wait_for_select(|checkpoint| {
19172                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
19173            })
19174            .await;
19175        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
19176        assert_eq!(Instant::now(), tick);
19177        tokio::time::advance(Duration::from_millis(1)).await;
19178        let mut started = false;
19179        wait_for("successful HTTP probe", || {
19180            started |= started_rx.try_recv().is_ok();
19181            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
19182                .then_some(())
19183        })
19184        .await;
19185        assert!(started, "added HTTP check must start at its first deadline");
19186        assert_eq!(
19187            actor.module.status().unwrap().health.status,
19188            SupervisorHealthStatus::Ok
19189        );
19190        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
19191        actor.health.http = None;
19192        let turns = actor.turns();
19193        actor
19194            .module
19195            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19196            .await
19197            .unwrap();
19198        let parked = actor
19199            .wait_for_select(|checkpoint| {
19200                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
19201            })
19202            .await;
19203        assert_eq!(parked.wake_after, None);
19204        let turns = actor.turns();
19205        tokio::time::advance(Duration::from_secs(60)).await;
19206        assert_eq!(
19207            actor.turns(),
19208            turns,
19209            "removed HTTP check must park the actor"
19210        );
19211        assert!(
19212            started_rx.try_recv().is_err(),
19213            "removed HTTP check must stay disarmed"
19214        );
19215        assert_eq!(
19216            actor.module.status().unwrap().health,
19217            ModuleHealthStatus::default()
19218        );
19219        peer.abort();
19220    }
19221}
19222
19223#[cfg(test)]
19224mod drain_busy_reprobe_tests {
19225    //! The drain's busy-gauge wait, driven against an in-process module
19226    //! connection whose health answers the test scripts. Time is paused, so
19227    //! every elapsed duration below is exact virtual time.
19228    use super::*;
19229    use std::collections::BTreeMap;
19230    use std::sync::Mutex as StdMutex;
19231    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
19232
19233    const MODULE_ID: &str = "drain-reprobe-module";
19234    const CONNECTION: u64 = 7;
19235    const GAUGE: &str = "runs_in_flight";
19236    /// The production default for both the health cadence and the drain
19237    /// ceiling, which is the pairing that used to cost a full ceiling.
19238    const CADENCE: Duration = Duration::from_secs(30);
19239    const CEILING: Duration = Duration::from_secs(30);
19240
19241    /// How the scripted module answers each `health.check`.
19242    #[derive(Clone, Copy)]
19243    enum Answer {
19244        /// Report the gauge busy until this instant, quiet from then on, after
19245        /// taking `latency` to answer.
19246        BusyUntil { until: Instant, latency: Duration },
19247        /// Answer at once, with metrics that leave the declared gauge out.
19248        OmitsGauge,
19249        /// Never answer.
19250        Silent,
19251    }
19252
19253    struct Drain {
19254        forwarding: Arc<ForwardingTable>,
19255        runtime: SupervisorRuntimeConfig,
19256        endpoint: crate::ModuleEndpointId,
19257        /// When each probe reached the module.
19258        probes: Arc<StdMutex<Vec<Instant>>>,
19259        responder: tokio::task::JoinHandle<()>,
19260    }
19261
19262    impl Drain {
19263        fn start(answer: Answer) -> Self {
19264            let registry = Arc::new(Registry::default());
19265            let forwarding = Arc::new(ForwardingTable::default());
19266            let health = HealthConfig {
19267                cadence: CADENCE,
19268                ..HealthConfig::default()
19269            };
19270            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
19271                .with_forwarding(forwarding.clone())
19272                .with_health_config(health)
19273                .runtime_config();
19274            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
19275            let endpoint = forwarding
19276                .register_module_connection(
19277                    ConnectionId::new(CONNECTION),
19278                    MODULE_ID.to_string(),
19279                    subc_protocol::PROTOCOL_VERSION,
19280                    Concurrency::ModuleManaged,
19281                    FrameSink::new(tx),
19282                )
19283                .unwrap();
19284            let probes = Arc::new(StdMutex::new(Vec::new()));
19285            let responder = tokio::spawn({
19286                let forwarding = forwarding.clone();
19287                let probes = probes.clone();
19288                async move {
19289                    while let Some(frame) = rx.recv().await {
19290                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
19291                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
19292                        probes.lock().unwrap().push(Instant::now());
19293                        let metrics = match answer {
19294                            Answer::BusyUntil { until, latency } => {
19295                                sleep(latency).await;
19296                                let busy = u64::from(Instant::now() < until);
19297                                serde_json::json!({ GAUGE: busy })
19298                            }
19299                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
19300                            Answer::Silent => continue,
19301                        };
19302                        // A probe abandoned at its deadline has no RPC left to
19303                        // complete, so a late answer may be refused.
19304                        let _ = forwarding.complete_module_control_rpc(
19305                            ConnectionId::new(CONNECTION),
19306                            frame.header.corr,
19307                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
19308                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19309                                status: HealthStatus::Ok,
19310                                detail: None,
19311                                metrics: Some(metrics),
19312                            }),
19313                        );
19314                    }
19315                }
19316            });
19317            Self {
19318                forwarding,
19319                runtime,
19320                endpoint,
19321                probes,
19322                responder,
19323            }
19324        }
19325
19326        /// Run one by-id drain wait with `ceiling`, returning its verdict and
19327        /// how long it took.
19328        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
19329            let (drained, elapsed, _) = self.run_traced(ceiling).await;
19330            (drained, elapsed)
19331        }
19332
19333        /// [`Self::run`], also returning what the drain's probes read.
19334        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
19335            let started = Instant::now();
19336            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
19337            let drained = wait_for_forwarding_quiescence(
19338                &self.forwarding,
19339                MODULE_ID,
19340                &self.runtime,
19341                self.endpoint,
19342                started + ceiling,
19343                &[GAUGE.to_string()],
19344                DrainScope::Active,
19345                &mut trace,
19346            )
19347            .await
19348            .unwrap();
19349            (drained, started.elapsed(), trace)
19350        }
19351
19352        fn probes(&self) -> Vec<Instant> {
19353            self.probes.lock().unwrap().clone()
19354        }
19355    }
19356
19357    impl Drop for Drain {
19358        fn drop(&mut self) {
19359            self.responder.abort();
19360        }
19361    }
19362
19363    #[tokio::test(start_paused = true)]
19364    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
19365        let drain = Drain::start(Answer::BusyUntil {
19366            until: Instant::now() + Duration::from_secs(1),
19367            latency: Duration::ZERO,
19368        });
19369        let (drained, elapsed) = drain.run(CEILING).await;
19370        assert!(
19371            drained,
19372            "gauges that read zero during the drain must let it finish drained, \
19373             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
19374        );
19375        assert!(
19376            elapsed < Duration::from_secs(3),
19377            "drain took {elapsed:?} after the gauges went quiet at 1s"
19378        );
19379        let probes = drain.probes();
19380        assert!(
19381            probes.len() >= 2,
19382            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
19383        );
19384    }
19385
19386    /// The same shape in real time on a multi-threaded runtime: the first
19387    /// probe is answered busy at once, within a millisecond of the drain
19388    /// starting, and later probes read idle. The drain must keep probing
19389    /// after that first instant answer and end soon after the gauge falls.
19390    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
19391    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
19392        let drain = Drain::start(Answer::BusyUntil {
19393            until: Instant::now() + Duration::from_millis(600),
19394            latency: Duration::ZERO,
19395        });
19396        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
19397        assert!(drained, "the gauge fell at 600ms: {trace}");
19398        assert!(
19399            elapsed < Duration::from_secs(3),
19400            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
19401        );
19402        let probes = drain.probes();
19403        assert!(
19404            probes.len() >= 2 && trace.probes as usize == probes.len(),
19405            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
19406        );
19407    }
19408
19409    #[tokio::test(start_paused = true)]
19410    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
19411        let started = Instant::now();
19412        let drain = Drain::start(Answer::BusyUntil {
19413            until: started + Duration::from_secs(3600),
19414            latency: Duration::ZERO,
19415        });
19416        let (drained, elapsed) = drain.run(CEILING).await;
19417        assert!(!drained, "a module busy throughout must not drain");
19418        assert!(
19419            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
19420            "a busy module is held until the ceiling and no longer: {elapsed:?}"
19421        );
19422        // Re-probing is paced by the interval, not a busy loop: at most one
19423        // probe per interval across the ceiling, plus the first at time zero.
19424        let probes = drain.probes();
19425        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
19426        assert!(
19427            probes.len() as u128 <= max_probes,
19428            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
19429            probes.len()
19430        );
19431        for pair in probes.windows(2) {
19432            assert!(
19433                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
19434                "consecutive probes {:?} apart",
19435                pair[1] - pair[0]
19436            );
19437        }
19438    }
19439
19440    #[tokio::test(start_paused = true)]
19441    async fn a_slow_probe_is_never_overlapped_by_the_next() {
19442        // Each answer takes longer than the re-probe interval; the next probe
19443        // must wait for the previous answer and then the interval.
19444        let latency = Duration::from_secs(1);
19445        let drain = Drain::start(Answer::BusyUntil {
19446            until: Instant::now() + Duration::from_secs(3600),
19447            latency,
19448        });
19449        let (drained, _) = drain.run(Duration::from_secs(5)).await;
19450        assert!(!drained);
19451        let probes = drain.probes();
19452        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
19453        for pair in probes.windows(2) {
19454            assert!(
19455                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
19456                "a probe started {:?} after the previous one, before it could finish",
19457                pair[1] - pair[0]
19458            );
19459        }
19460    }
19461
19462    #[tokio::test(start_paused = true)]
19463    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
19464        let drain = Drain::start(Answer::Silent);
19465        let ceiling = Duration::from_secs(2);
19466        let (drained, elapsed) = drain.run(ceiling).await;
19467        assert!(!drained, "an unanswered probe counts as busy");
19468        assert!(
19469            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
19470            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
19471        );
19472        assert_eq!(
19473            drain.probes().len(),
19474            1,
19475            "the unanswered probe holds the only slot until the ceiling"
19476        );
19477    }
19478
19479    /// Every event logged while it is installed, as `message` plus fields.
19480    #[derive(Clone, Default)]
19481    struct LogCapture {
19482        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
19483    }
19484
19485    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
19486        fn on_event(
19487            &self,
19488            event: &tracing::Event<'_>,
19489            _context: tracing_subscriber::layer::Context<'_, S>,
19490        ) {
19491            struct Fields(BTreeMap<String, String>);
19492            impl tracing::field::Visit for Fields {
19493                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
19494                    self.0
19495                        .insert(field.name().to_string(), format!("{value:?}"));
19496                }
19497            }
19498            let mut fields = Fields(BTreeMap::new());
19499            event.record(&mut fields);
19500            self.events.lock().unwrap().push(fields.0);
19501        }
19502    }
19503
19504    impl LogCapture {
19505        fn install(&self) -> tracing::subscriber::DefaultGuard {
19506            use tracing_subscriber::layer::SubscriberExt;
19507            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
19508        }
19509
19510        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
19511            self.events
19512                .lock()
19513                .unwrap()
19514                .iter()
19515                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
19516                .cloned()
19517                .collect()
19518        }
19519    }
19520
19521    /// A module that declares a busy gauge but leaves it out of its health
19522    /// answers holds a drain to its ceiling. The timeout line must say so by
19523    /// name, and the drain must warn about the missing gauge once, not on
19524    /// every probe.
19525    #[tokio::test(start_paused = true)]
19526    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
19527        let capture = LogCapture::default();
19528        let _guard = capture.install();
19529        let drain = Drain::start(Answer::OmitsGauge);
19530        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
19531        assert!(!drained, "a gauge never reported cannot read quiet");
19532        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
19533
19534        let missing = capture.with_message("lacks a declared busy gauge");
19535        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
19536        assert_eq!(
19537            missing[0].get("missing_gauges").map(String::as_str),
19538            Some(GAUGE)
19539        );
19540
19541        warn_drain_timed_out(
19542            MODULE_ID,
19543            Duration::from_secs(2),
19544            RouteCloseReason::Restart,
19545            &Default::default(),
19546            &trace,
19547        );
19548        let timeouts = capture.with_message("route drain timed out");
19549        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
19550        let gauges = &timeouts[0]["busy_gauges"];
19551        assert!(
19552            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
19553            "the timeout line must name the gauge and why it read busy: {gauges}"
19554        );
19555        assert!(
19556            gauges.ends_with("; no value ever read"),
19557            "no value was ever read for the gauge: {gauges}"
19558        );
19559        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
19560    }
19561
19562    /// The trace names a busy value and when it was read. For a probe that
19563    /// never got an answer, it names the probe's error kind and says that no
19564    /// value was ever read.
19565    #[tokio::test(start_paused = true)]
19566    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
19567        let busy = Drain::start(Answer::BusyUntil {
19568            until: Instant::now() + Duration::from_secs(3600),
19569            latency: Duration::ZERO,
19570        });
19571        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
19572        let rendered = trace.to_string();
19573        assert!(
19574            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
19575                && rendered.contains("; last value non_zero(1) @"),
19576            "{rendered}"
19577        );
19578
19579        let silent = Drain::start(Answer::Silent);
19580        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
19581        // The probe is capped at the 1 s ceiling, and its reading is stamped
19582        // when it gave up.
19583        assert_eq!(
19584            trace.to_string(),
19585            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
19586        );
19587        assert_eq!(trace.probes, 1);
19588    }
19589
19590    /// A gauge value that is not a count reads busy and is shown as sent.
19591    #[test]
19592    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
19593        let gauges = [GAUGE.to_string()];
19594        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
19595        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
19596        assert_eq!(
19597            busy_gauge_observation(&verdicts),
19598            BusyGaugeObservation::Busy
19599        );
19600        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
19601        assert_eq!(
19602            busy_gauge_observation(&zero),
19603            BusyGaugeObservation::Quiescent
19604        );
19605        let none = gauge_verdicts(None, &gauges);
19606        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
19607    }
19608}
19609
19610#[cfg(test)]
19611mod health_evidence_tests {
19612    use super::{HealthProbeError, HealthProbeEvidence};
19613    use std::collections::HashSet;
19614
19615    /// The evidential asymmetry, asserted rather than described.
19616    ///
19617    /// Exactly ONE observation is proof a module cannot serve, and the one that
19618    /// fires under CPU starvation is not it. Before the split, all fifteen
19619    /// construction sites collapsed into a single String, so a timeout carried the
19620    /// same weight as a dead lane -- which is how a healthy module was restarted
19621    /// three times in one day.
19622    #[test]
19623    fn only_a_dead_lane_is_proof_of_death() {
19624        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
19625        // Three non-proof classes, each for a different reason: silence is
19626        // consistent with health, a bad answer proves the module ALIVE, and a
19627        // daemon-side fault never reached the module at all.
19628        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
19629        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
19630        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
19631    }
19632
19633    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
19634    ///
19635    /// A shared label renders two different observations identically in the line an
19636    /// operator reads after an unexplained restart -- the exact confusion this
19637    /// change removes.
19638    #[test]
19639    fn every_evidence_class_has_a_distinct_label() {
19640        let labels = [
19641            HealthProbeError::lane_dead("").label(),
19642            HealthProbeError::no_answer("").label(),
19643            HealthProbeError::bad_answer("").label(),
19644            HealthProbeError::misconfigured("").label(),
19645        ];
19646        let unique: HashSet<_> = labels.iter().collect();
19647        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
19648    }
19649
19650    /// The class is additional information, not a replacement.
19651    ///
19652    /// An operator needs both "this was silence" and the specific text saying how
19653    /// long we waited; a classification that swallowed the message would trade one
19654    /// missing distinction for another.
19655    #[test]
19656    fn classification_preserves_the_original_message() {
19657        let err = HealthProbeError::no_answer("module did not answer within 5s");
19658        assert_eq!(err.to_string(), "module did not answer within 5s");
19659        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19660    }
19661}
19662
19663#[cfg(test)]
19664mod health_tombstone_tests {
19665    use std::{path::PathBuf, sync::Arc, time::Duration};
19666
19667    use subc_protocol::{
19668        manifest::Concurrency,
19669        session::{HealthStatus, ModuleControlResponse},
19670    };
19671    use tokio::sync::mpsc;
19672
19673    use super::{
19674        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
19675        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
19676    };
19677    use crate::{
19678        control::ControlHandler,
19679        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
19680        registry::{ConnectionId, Registry},
19681        router::FrameSink,
19682    };
19683
19684    struct ProbeHarness {
19685        spec: ModuleSpec,
19686        runtime: SupervisorRuntimeConfig,
19687        forwarding: Arc<ForwardingTable>,
19688        module_connection: ConnectionId,
19689        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19690        handler: ControlHandler,
19691        module: super::SupervisedModule,
19692    }
19693
19694    fn probe_harness() -> ProbeHarness {
19695        let registry = Arc::new(Registry::default());
19696        let forwarding = Arc::new(ForwardingTable::default());
19697        let supervisor_handle = super::SupervisorHandle::new();
19698        let health = HealthConfig {
19699            http: None,
19700            cadence: Duration::from_secs(30),
19701            deadline: Duration::from_secs(5),
19702            failure_threshold: 3,
19703            on_degraded: HealthAction::Report,
19704            on_failing: HealthAction::Report,
19705            critical: false,
19706        };
19707        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
19708            .with_forwarding(Arc::clone(&forwarding))
19709            .with_handle(supervisor_handle.clone())
19710            .with_health_config(health);
19711        let spec = ModuleSpec {
19712            script: None,
19713            module_id: "late-health-module".to_string(),
19714            program: PathBuf::from("disabled-module"),
19715            args: Vec::new(),
19716            env: Vec::new(),
19717            reserved: false,
19718            reserved_prefixes: Vec::new(),
19719            protocol: ModuleProtocol::Subc,
19720            overlap: Default::default(),
19721            carry_containment: false,
19722        };
19723        let module = supervisor
19724            .supervise_configured(spec.clone(), false)
19725            .unwrap();
19726        let runtime = supervisor.runtime_config();
19727        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
19728            .with_supervisor(supervisor_handle);
19729        let module_connection = ConnectionId::new(700);
19730        let (module_tx, module_rx) = mpsc::channel(8);
19731        forwarding
19732            .register_module_connection(
19733                module_connection,
19734                spec.module_id.clone(),
19735                subc_protocol::PROTOCOL_VERSION,
19736                Concurrency::ModuleManaged,
19737                FrameSink::new(module_tx),
19738            )
19739            .unwrap();
19740
19741        ProbeHarness {
19742            spec,
19743            runtime,
19744            forwarding,
19745            module_connection,
19746            module_rx,
19747            handler,
19748            module,
19749        }
19750    }
19751
19752    async fn finish_after(
19753        harness: &mut ProbeHarness,
19754        stall: Duration,
19755    ) -> ModuleControlRpcCompletion {
19756        assert!(stall > harness.runtime.health.deadline);
19757        let deadline = harness.runtime.health.deadline;
19758        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19759        let answer = async {
19760            let frame = harness.module_rx.recv().await.expect("health.check frame");
19761            tokio::time::advance(deadline).await;
19762            tokio::task::yield_now().await;
19763            tokio::time::advance(stall - deadline).await;
19764            harness
19765                .forwarding
19766                .complete_module_control_rpc(
19767                    harness.module_connection,
19768                    frame.header.corr,
19769                    Some("health.check"),
19770                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19771                        status: HealthStatus::Ok,
19772                        detail: None,
19773                        metrics: None,
19774                    }),
19775                )
19776                .unwrap()
19777        };
19778        let (probe_result, completion) = tokio::join!(probe, answer);
19779        let err = probe_result.expect_err("probe must miss its deadline");
19780        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19781        completion
19782    }
19783
19784    async fn time_out_without_answer(harness: &mut ProbeHarness) {
19785        let deadline = harness.runtime.health.deadline;
19786        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19787        let exhaust_deadline = async {
19788            let _frame = harness.module_rx.recv().await.expect("health.check frame");
19789            tokio::time::advance(deadline).await;
19790            tokio::task::yield_now().await;
19791        };
19792        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
19793        let err = probe_result.expect_err("probe must miss its deadline");
19794        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19795    }
19796
19797    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
19798        let registry = Arc::clone(&harness.module.inner.registry);
19799        let snapshot = Arc::clone(&harness.module.inner.snapshot);
19800        let process_liveness = super::SupervisorProcessLiveness::default();
19801        let mut child = None;
19802        let cycle = super::run_health_probe_cycle(
19803            &harness.spec,
19804            &harness.runtime,
19805            &registry,
19806            &process_liveness,
19807            &snapshot,
19808            &mut child,
19809        );
19810        let peer = async {
19811            let frame = harness.module_rx.recv().await.expect("health.check frame");
19812            if answer {
19813                harness
19814                    .forwarding
19815                    .complete_module_control_rpc(
19816                        harness.module_connection,
19817                        frame.header.corr,
19818                        Some("health.check"),
19819                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19820                            status: HealthStatus::Ok,
19821                            detail: None,
19822                            metrics: Some(serde_json::json!({"ready": true})),
19823                        }),
19824                    )
19825                    .unwrap();
19826            } else {
19827                tokio::time::advance(harness.runtime.health.deadline).await;
19828                tokio::task::yield_now().await;
19829            }
19830        };
19831        tokio::join!(cycle, peer);
19832    }
19833
19834    #[tokio::test(start_paused = true)]
19835    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
19836        let mut harness = probe_harness();
19837        // Drive the probe cycle directly with an in-memory wire peer. Stop the
19838        // disabled module's monitor so only this test owns lifecycle transitions;
19839        // no OS process is launched, and a restart is observed at scheduling.
19840        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
19841        monitor.abort();
19842        let _ = monitor.await;
19843        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
19844            state.enabled = true;
19845            state.state = super::ModuleState::Running;
19846            state.process_alive = true;
19847        })
19848        .unwrap();
19849
19850        run_probe_cycle(&mut harness, true).await;
19851        assert_eq!(
19852            harness.module.status().unwrap().health.status,
19853            super::SupervisorHealthStatus::Ok
19854        );
19855
19856        for failures in 1..harness.runtime.health.failure_threshold {
19857            run_probe_cycle(&mut harness, false).await;
19858            let status = harness.module.status().unwrap();
19859            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19860            assert_eq!(status.health.consecutive_failures, failures);
19861            assert!(status.health.last_probe_ms.is_some());
19862            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
19863            assert!(status.health.metrics.is_none());
19864            assert_eq!(status.state, super::ModuleState::Running);
19865            assert!(status.process_alive);
19866            assert_eq!(status.restart_count, 0);
19867            assert_eq!(status.lifetime_restarts, 0);
19868            assert!(status.health.last_action.is_none());
19869            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19870        }
19871
19872        run_probe_cycle(&mut harness, true).await;
19873        let recovered = harness.module.status().unwrap();
19874        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
19875        assert_eq!(recovered.health.consecutive_failures, 0);
19876        assert!(recovered.health.detail.is_none());
19877        assert_eq!(
19878            recovered.health.metrics,
19879            Some(serde_json::json!({"ready": true}))
19880        );
19881        assert_eq!(recovered.lifetime_restarts, 0);
19882
19883        for failures in 1..=harness.runtime.health.failure_threshold {
19884            run_probe_cycle(&mut harness, false).await;
19885            let status = harness.module.status().unwrap();
19886            assert_eq!(status.health.consecutive_failures, failures);
19887            if failures < harness.runtime.health.failure_threshold {
19888                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19889                assert_eq!(status.state, super::ModuleState::Running);
19890                assert_eq!(status.lifetime_restarts, 0);
19891                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19892            } else {
19893                assert_eq!(
19894                    status.health.status,
19895                    super::SupervisorHealthStatus::Unresponsive
19896                );
19897                assert_eq!(status.state, super::ModuleState::Restarting);
19898                assert_eq!(status.restart_count, 1);
19899                assert_eq!(status.lifetime_restarts, 1);
19900                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
19901                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
19902            }
19903        }
19904    }
19905
19906    #[tokio::test(start_paused = true)]
19907    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
19908        let mut harness = probe_harness();
19909
19910        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
19911        let first_latency = match &first {
19912            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19913            other => panic!("late answer was not retained: {other:?}"),
19914        };
19915        assert!(harness.handler.observe_module_control_completion(first));
19916
19917        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
19918        let second_latency = match &second {
19919            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19920            other => panic!("late answer was not retained: {other:?}"),
19921        };
19922        assert!(harness.handler.observe_module_control_completion(second));
19923
19924        assert_eq!(first_latency, Duration::from_secs(8));
19925        assert_eq!(
19926            second_latency - first_latency,
19927            Duration::from_secs(3),
19928            "latency must grow linearly with the additional stall"
19929        );
19930        let health = harness.module.status().unwrap().health;
19931        assert_eq!(health.late_answer_count, 2);
19932        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
19933    }
19934
19935    /// A module that answers every probe late must never march to the kill
19936    /// threshold: the late answer proves it is alive, so it must clear the miss
19937    /// streak the timeout recorded. Without the reset, a CPU-starved module
19938    /// that serves every probe seconds past the deadline accumulates
19939    /// `consecutive_failures` to the threshold and is killed — the exact
19940    /// sequence from the 2026-08-14 aft disable, where the daemon logged
19941    /// "proves the module is alive" five times while counting five misses.
19942    #[tokio::test(start_paused = true)]
19943    async fn late_answer_clears_the_consecutive_failure_streak() {
19944        let mut harness = probe_harness();
19945
19946        // Timeout recorded first: the probe path saw no answer in time.
19947        time_out_without_answer(&mut harness).await;
19948        harness
19949            .module
19950            .record_health_probe_failure_for_test("[no-answer] test miss")
19951            .unwrap();
19952        assert_eq!(
19953            harness.module.status().unwrap().health.consecutive_failures,
19954            1,
19955            "precondition: the miss must be on the streak before the late answer"
19956        );
19957
19958        // The stalled reply then lands: proof of life.
19959        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
19960        assert!(matches!(
19961            late,
19962            ModuleControlRpcCompletion::LateHealthAnswer { .. }
19963        ));
19964        assert!(harness.handler.observe_module_control_completion(late));
19965
19966        let health = harness.module.status().unwrap().health;
19967        assert_eq!(
19968            health.consecutive_failures, 0,
19969            "a late answer is an answer: the streak must reset"
19970        );
19971        assert_eq!(health.late_answer_count, 1);
19972    }
19973
19974    #[tokio::test(start_paused = true)]
19975    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
19976        let mut harness = probe_harness();
19977
19978        for _ in 0..20 {
19979            time_out_without_answer(&mut harness).await;
19980            assert_eq!(
19981                harness.forwarding.health_probe_tombstone_count().unwrap(),
19982                1
19983            );
19984        }
19985    }
19986}
19987
19988#[cfg(test)]
19989mod child_env_tests {
19990    #[cfg(windows)]
19991    use super::{
19992        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
19993    };
19994    #[cfg(windows)]
19995    use std::sync::{Arc, OnceLock};
19996    #[cfg(windows)]
19997    #[test]
19998    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
19999        let spec = spec(Vec::new());
20000        let mut command = Command::new("unused");
20001        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
20002            .unwrap()
20003            .unwrap();
20004        let name = command
20005            .as_std()
20006            .get_envs()
20007            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
20008            .and_then(|(_, value)| value)
20009            .unwrap();
20010        assert_eq!(name, OsStr::new(handoff.name()));
20011        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
20012        assert!(command.as_std().get_envs().any(|(key, value)| key
20013            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
20014            && value == Some(OsStr::new("env"))));
20015        assert!(command
20016            .as_std()
20017            .get_envs()
20018            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
20019    }
20020
20021    #[cfg(windows)]
20022    #[test]
20023    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
20024        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
20025        snapshot.process_alive = true;
20026        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
20027        assert_eq!(
20028            observed_launch_nonce_source(&snapshot).as_deref(),
20029            Some("env")
20030        );
20031        let consumed = Arc::new(OnceLock::new());
20032        snapshot.pipe_consumed = Some(consumed.clone());
20033        assert_eq!(
20034            observed_launch_nonce_source(&snapshot).as_deref(),
20035            Some("env")
20036        );
20037        consumed.set(()).unwrap();
20038        assert_eq!(
20039            observed_launch_nonce_source(&snapshot).as_deref(),
20040            Some("pipe")
20041        );
20042        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
20043        assert_eq!(
20044            observed_launch_nonce_source(&snapshot).as_deref(),
20045            Some("env")
20046        );
20047        clear_current_process_facts(&mut snapshot);
20048        assert_eq!(observed_launch_nonce_source(&snapshot), None);
20049    }
20050    use super::{
20051        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
20052        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
20053        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
20054    };
20055    use std::{ffi::OsStr, path::PathBuf};
20056    use tokio::process::Command;
20057
20058    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
20059        ModuleSpec {
20060            script: None,
20061            module_id: "env-plan".to_string(),
20062            program: PathBuf::from("/nonexistent"),
20063            args: Vec::new(),
20064            env,
20065            reserved: false,
20066            reserved_prefixes: Vec::new(),
20067            protocol: ModuleProtocol::Subc,
20068            overlap: Default::default(),
20069            carry_containment: false,
20070        }
20071    }
20072
20073    #[test]
20074    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
20075        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
20076        let digest = LaunchSpecDigest::from_spec(&base);
20077        let mut reordered = base.clone();
20078        reordered.env.reverse();
20079        reordered.module_id = "different-id".into();
20080        reordered.reserved = true;
20081        reordered.overlap = Default::default();
20082        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
20083        let mut delimiter_collision = base.clone();
20084        delimiter_collision.env[0] = ("A=B".into(), "C".into());
20085        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
20086        let mut changed = base.clone();
20087        changed.protocol = ModuleProtocol::None;
20088        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20089        changed = base.clone();
20090        changed.args.push("--flag".into());
20091        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20092        changed = base.clone();
20093        changed.program = PathBuf::from("/other");
20094        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20095        changed = base.clone();
20096        changed.env[1].1 = "other".into();
20097        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20098        assert!(!format!("{digest:?}").contains("B=C"));
20099        assert!(!format!("{digest:?}").contains("[u8"));
20100    }
20101
20102    #[cfg(unix)]
20103    #[test]
20104    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
20105        use std::os::unix::ffi::OsStringExt;
20106        let mut left = spec(Vec::new());
20107        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
20108        let mut right = left.clone();
20109        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
20110        assert_eq!(
20111            left.program.to_string_lossy(),
20112            right.program.to_string_lossy()
20113        );
20114        assert_ne!(
20115            LaunchSpecDigest::from_spec(&left),
20116            LaunchSpecDigest::from_spec(&right)
20117        );
20118    }
20119
20120    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
20121    /// one still gets its own.
20122    ///
20123    /// This is the narrow goal `env_clear()` was reached for, and the reason the
20124    /// fix is `env_remove` rather than deleting the line: an operator's ambient
20125    /// filter silently becoming an unconfigured module's log level is a real
20126    /// defect, just a much smaller one than clearing the environment.
20127    ///
20128    /// Asserted on the command plan rather than a spawned child because proving
20129    /// the ABSENCE of an inherited variable needs the parent's environment
20130    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
20131    /// removal as `(key, None)`, which is exactly the distinction wanted: not
20132    /// "absent because nobody set it" but "explicitly unset for the child".
20133    #[test]
20134    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
20135        let mut command = Command::new("/nonexistent");
20136        apply_child_env(&mut command, &spec(Vec::new()));
20137        let removed = command
20138            .as_std()
20139            .get_envs()
20140            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
20141        assert!(
20142            removed,
20143            "ambient CK_LOG must be explicitly removed for an unconfigured module"
20144        );
20145
20146        let mut configured = Command::new("/nonexistent");
20147        apply_child_env(
20148            &mut configured,
20149            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
20150        );
20151        let effective = configured
20152            .as_std()
20153            .get_envs()
20154            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
20155            .last()
20156            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
20157        assert_eq!(
20158            effective,
20159            Some(Some("debug".to_string())),
20160            "a module's configured CK_LOG must survive the ambient removal"
20161        );
20162    }
20163
20164    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
20165    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
20166    /// the same reason as the CK_LOG test above.
20167    ///
20168    /// The argument is the load-bearing half: a stock binary exits on an
20169    /// unknown flag before it listens, so with `--subc` appended the mode
20170    /// could not supervise the one process it exists for. Found by the first
20171    /// conformance run (nats-server: `flag provided but not defined: -subc`).
20172    #[test]
20173    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
20174        let connection_file = std::path::Path::new("/run/subc-connection.json");
20175        let handle = SupervisorHandle::new();
20176
20177        let mut none_spec = spec(Vec::new());
20178        none_spec.protocol = ModuleProtocol::None;
20179        let mut none = Command::new("/nonexistent");
20180        let none_handoff =
20181            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
20182                .expect("protocol-none spawn args apply");
20183        assert!(
20184            none_handoff.is_none(),
20185            "protocol:none spawn must not receive a nonce descriptor"
20186        );
20187        #[cfg(windows)]
20188        assert!(!none.as_std().get_envs().any(|(key, value)| key
20189            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
20190            && value.is_some()));
20191        assert!(
20192            !none.as_std().get_envs().any(|(key, value)| key
20193                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
20194                && value.is_some()),
20195            "protocol:none spawn must not name a nonce descriptor"
20196        );
20197        let none_args: Vec<String> = none
20198            .as_std()
20199            .get_args()
20200            .map(|a| a.to_string_lossy().into_owned())
20201            .collect();
20202        assert!(
20203            !none_args.iter().any(|a| a == SUBC_ARG),
20204            "protocol:none argv must not carry --subc; got {none_args:?}"
20205        );
20206        let none_has_nonce = none
20207            .as_std()
20208            .get_envs()
20209            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
20210        assert!(
20211            !none_has_nonce,
20212            "protocol:none spawn must not receive a launch nonce"
20213        );
20214        let none_has_module_id = none
20215            .as_std()
20216            .get_envs()
20217            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
20218        assert!(
20219            none_has_module_id,
20220            "SUBC_MODULE_ID is inert and stays on every path"
20221        );
20222        assert!(
20223            handle.spawn_nonce(&none_spec.module_id).is_none(),
20224            "no nonce record for a process that will never present one"
20225        );
20226
20227        // Control: the subc-wire path is unchanged by the branch above.
20228        let wire_spec = spec(Vec::new());
20229        let mut wire = Command::new("/nonexistent");
20230        let wire_handoff =
20231            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
20232                .expect("subc-wire spawn args apply");
20233        let wire_fd_env = wire
20234            .as_std()
20235            .get_envs()
20236            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
20237            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
20238        #[cfg(unix)]
20239        assert_eq!(
20240            wire_fd_env,
20241            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
20242            "a subc-wire spawn names the pipe it will receive at descriptor 3"
20243        );
20244        #[cfg(windows)]
20245        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
20246        #[cfg(not(any(unix, windows)))]
20247        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
20248        let wire_args: Vec<String> = wire
20249            .as_std()
20250            .get_args()
20251            .map(|a| a.to_string_lossy().into_owned())
20252            .collect();
20253        assert_eq!(
20254            wire_args,
20255            vec![
20256                SUBC_ARG.to_string(),
20257                connection_file.to_string_lossy().into_owned()
20258            ],
20259            "a subc-wire spawn still carries --subc <path>"
20260        );
20261        assert_eq!(
20262            wire.as_std()
20263                .get_envs()
20264                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
20265            !cfg!(unix),
20266            "only Windows supplies the environment nonce"
20267        );
20268        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
20269    }
20270
20271    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
20272    /// spec tries to set it; only a swap candidate carries it.
20273    ///
20274    /// "Set it only on candidates" is not enough, because spawn applies the
20275    /// spec's env verbatim and the daemon's own environment is inherited: either
20276    /// could hand a plain restart the swap role, and a module reading it would
20277    /// warm on its long swap budget while callers wait. Asserted as an explicit
20278    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
20279    /// test above gives.
20280    #[test]
20281    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
20282        let role = |command: &Command| {
20283            command
20284                .as_std()
20285                .get_envs()
20286                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
20287                .last()
20288                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
20289        };
20290        let forged = spec(vec![(
20291            SUBC_SPAWN_ROLE_ENV.to_string(),
20292            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
20293        )]);
20294
20295        let mut plain = Command::new("/nonexistent");
20296        apply_child_env(&mut plain, &forged);
20297        apply_spawn_role(&mut plain, SpawnRole::Plain);
20298        assert_eq!(
20299            role(&plain),
20300            Some(None),
20301            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
20302        );
20303
20304        let mut candidate = Command::new("/nonexistent");
20305        apply_child_env(&mut candidate, &spec(Vec::new()));
20306        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
20307        assert_eq!(
20308            role(&candidate),
20309            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
20310        );
20311    }
20312
20313    /// Daemon-private capture retention keys never reach the child.
20314    ///
20315    /// cortexkit-log exposes retention as a Rust struct with no environment
20316    /// names, so these entries are supervisor metadata. Passing them through
20317    /// would invent a public child-process contract by accident.
20318    #[test]
20319    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
20320        let mut command = Command::new("/nonexistent");
20321        apply_child_env(
20322            &mut command,
20323            &spec(vec![
20324                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
20325                ("KEPT".to_string(), "yes".to_string()),
20326            ]),
20327        );
20328        let keys: Vec<String> = command
20329            .as_std()
20330            .get_envs()
20331            .filter(|(_, value)| value.is_some())
20332            .map(|(key, _)| key.to_string_lossy().into_owned())
20333            .collect();
20334        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
20335        assert!(
20336            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
20337            "daemon-private capture key leaked to the child: {keys:?}"
20338        );
20339    }
20340}
20341
20342#[cfg(test)]
20343mod jitter_tests {
20344    use super::jittered_health_delay;
20345    use std::{collections::HashSet, time::Duration};
20346
20347    /// Module ids drawn from a real fleet, so the dispersal claim is about names
20348    /// that actually occur rather than invented ones.
20349    ///
20350    /// This is a SAMPLE, not a registry: the property under test is that distinct
20351    /// ids disperse, which holds for any set of distinct strings. Several entries
20352    /// are already historical (modules get renamed), and that costs nothing here --
20353    /// but it means a reader must not mistake this for the live module set, and a
20354    /// rename sweep will match it without there being anything to change.
20355    const FLEET: [&str; 14] = [
20356        "aft",
20357        "alfonso-core",
20358        "magic-context",
20359        "broca",
20360        "thalamus",
20361        "quota",
20362        "engram",
20363        "plexus",
20364        "cerebellum",
20365        "astrocyte",
20366        "synapse",
20367        "subc-mcp",
20368        "cortexkit-credentials",
20369        "subc-federation",
20370    ];
20371
20372    /// Probes must not converge after a fleet-wide restart.
20373    ///
20374    /// This is the property the jitter exists for: every module reconnects at
20375    /// once, and without dispersal all fourteen would then probe on the same
20376    /// tick forever. Nothing failed visibly when this went untested -- a
20377    /// convergent fleet still probes correctly, just in a burst, so the symptom
20378    /// is a periodic load spike that looks like whatever else is running.
20379    #[test]
20380    fn probe_delays_disperse_across_the_fleet() {
20381        let cadence = Duration::from_secs(30);
20382        let delays: HashSet<Duration> = FLEET
20383            .iter()
20384            .map(|id| jittered_health_delay(id, 0, cadence))
20385            .collect();
20386        assert_eq!(
20387            delays.len(),
20388            FLEET.len(),
20389            "every supervised module must land on its own probe offset"
20390        );
20391    }
20392
20393    /// The offset may only ever DELAY a probe, never bring it forward.
20394    ///
20395    /// A delay below the cadence would probe a module more often than
20396    /// configured, which is the opposite of what an operator asked for and
20397    /// would tighten the failure budget without anyone changing it.
20398    #[test]
20399    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
20400        let cadence = Duration::from_secs(30);
20401        let span = cadence / 10;
20402        for id in FLEET {
20403            for probe_index in 0..8 {
20404                let delay = jittered_health_delay(id, probe_index, cadence);
20405                assert!(
20406                    delay >= cadence,
20407                    "{id}#{probe_index}: jitter must not shorten the cadence"
20408                );
20409                assert!(
20410                    delay < cadence + span,
20411                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
20412                );
20413            }
20414        }
20415    }
20416
20417    /// A module keeps its offset across daemon restarts.
20418    ///
20419    /// The delay is derived rather than randomised precisely so a restart does
20420    /// not re-roll every module into a fresh chance of collision. A random
20421    /// source would satisfy the dispersal test above and quietly lose this.
20422    #[test]
20423    fn a_module_offset_is_stable_across_restarts() {
20424        let cadence = Duration::from_secs(30);
20425        for id in FLEET {
20426            assert_eq!(
20427                jittered_health_delay(id, 0, cadence),
20428                jittered_health_delay(id, 0, cadence),
20429                "{id}: the same module and probe index must produce the same offset"
20430            );
20431        }
20432    }
20433
20434    /// A zero cadence disables probing rather than producing a busy loop.
20435    #[test]
20436    fn zero_cadence_yields_zero_delay() {
20437        assert_eq!(
20438            jittered_health_delay("aft", 0, Duration::ZERO),
20439            Duration::ZERO
20440        );
20441    }
20442}
20443
20444#[cfg(all(test, target_os = "linux"))]
20445mod cgroup_placement_tests {
20446    use super::{
20447        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
20448        SuperviseError, SupervisedChild,
20449    };
20450    use crate::stderr_tail::{StderrRing, StderrTailConfig};
20451    use cortexkit_test_support::ScratchDir;
20452    use std::{
20453        fs, io,
20454        path::{Path, PathBuf},
20455        sync::{Arc, Mutex},
20456    };
20457    use tokio::process::Command;
20458
20459    #[tokio::test]
20460    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
20461        use super::*;
20462        let dir = ScratchDir::new("unique-spawn-cgroups");
20463        let root = PathBuf::from(format!(
20464            "/sys/fs/cgroup/subc-unique-test-{}-{}",
20465            std::process::id(),
20466            unix_ms_now()
20467        ));
20468        if let Err(error) = fs::create_dir(&root) {
20469            assert!(
20470                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
20471                "required cgroup test cannot execute: {error}"
20472            );
20473            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
20474            return;
20475        }
20476        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
20477        let group_count = || {
20478            fs::read_dir(root.join("subc-modules"))
20479                .unwrap()
20480                .map(|entry| entry.unwrap().file_type().unwrap())
20481                .filter(|kind| kind.is_dir())
20482                .count()
20483        };
20484        let supervisor =
20485            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
20486                .with_cgroup_placement(Some(placement.clone()));
20487        let runtime = supervisor.runtime_config();
20488        let mut spec = ModuleSpec {
20489            script: None,
20490            module_id: "unique-spawn".into(),
20491            program: PathBuf::from("/bin/sleep"),
20492            args: vec!["60".into()],
20493            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
20494                .into_iter()
20495                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
20496                .collect(),
20497            reserved: false,
20498            reserved_prefixes: vec![],
20499            protocol: ModuleProtocol::None,
20500            overlap: Default::default(),
20501            carry_containment: false,
20502        };
20503        let spawn = |spec: &ModuleSpec| {
20504            spawn_child(
20505                spec,
20506                None,
20507                None,
20508                &runtime.stderr_ring,
20509                None,
20510                &runtime.child_roster,
20511                runtime.spawn_delivery(),
20512                Some(&placement),
20513                runtime.register_timeout,
20514            )
20515            .unwrap()
20516        };
20517        let mut live = spawn(&spec);
20518        for _ in 0..3 {
20519            // A new process can enter the old slot while retirement is pending.
20520            let next = spawn(&spec);
20521            assert_ne!(live.module_id, next.module_id);
20522            live.start_kill().unwrap();
20523            live.wait().await.unwrap();
20524            live = next;
20525            assert!(
20526                live.child.try_wait().unwrap().is_none(),
20527                "retiring the old slot must not kill the replacement"
20528            );
20529            assert_eq!(
20530                group_count(),
20531                1,
20532                "only the live spawn's cgroup should remain"
20533            );
20534        }
20535        supervisor.begin_daemon_shutdown();
20536        let reap = tokio::spawn(async move {
20537            live.wait().await.unwrap();
20538        });
20539        supervisor
20540            .end_children_for_daemon_shutdown(false, std::future::pending())
20541            .await;
20542        reap.await.unwrap();
20543        assert_eq!(group_count(), 0);
20544        // A normal exit uses the same tree-cleanup path as a killed spawn.
20545        spec.program = PathBuf::from("/bin/true");
20546        spec.args.clear();
20547        let fresh_roster = ChildRoster::default();
20548        let mut short = spawn_child(
20549            &spec,
20550            None,
20551            None,
20552            &runtime.stderr_ring,
20553            None,
20554            &fresh_roster,
20555            None,
20556            Some(&placement),
20557            runtime.register_timeout,
20558        )
20559        .unwrap();
20560        short.wait().await.unwrap();
20561        assert_eq!(group_count(), 0);
20562        spec.module_id = "_".repeat(255);
20563        let mut long_id = spawn_child(
20564            &spec,
20565            None,
20566            None,
20567            &runtime.stderr_ring,
20568            None,
20569            &fresh_roster,
20570            None,
20571            Some(&placement),
20572            runtime.register_timeout,
20573        )
20574        .unwrap();
20575        long_id.wait().await.unwrap();
20576        assert_eq!(
20577            group_count(),
20578            0,
20579            "valid long module IDs must not exceed cgroup NAME_MAX"
20580        );
20581        fs::remove_dir(root.join("subc-modules")).unwrap();
20582        fs::remove_dir(root).unwrap();
20583        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
20584    }
20585
20586    #[test]
20587    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
20588        let path = Path::new("/definitely-missing-subc-cgroup");
20589        let mut command = Command::new("true");
20590        let error = apply_cgroup_placement(
20591            &mut command,
20592            &ModuleSpec {
20593                script: None,
20594                module_id: "broken-cgroup".to_string(),
20595                program: PathBuf::from("true"),
20596                args: Vec::new(),
20597                env: Vec::new(),
20598                reserved: false,
20599                reserved_prefixes: Vec::new(),
20600                protocol: ModuleProtocol::Subc,
20601                overlap: Default::default(),
20602                carry_containment: false,
20603            },
20604            path,
20605        )
20606        .expect_err("a parent cgroup open failure must reject the supervised spawn");
20607        let reason = error.to_string();
20608
20609        assert!(
20610            matches!(error, SuperviseError::Cgroup { .. }),
20611            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
20612        );
20613        assert!(
20614            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
20615            "parent cgroup open failure must name cgroup.procs: {reason}"
20616        );
20617    }
20618
20619    #[tokio::test]
20620    async fn reaping_a_child_removes_its_empty_module_cgroup() {
20621        let root = ScratchDir::new("supervisor-reap-cgroup");
20622        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20623        let placement = subc_cgroup::prepare_at(&root)
20624            .expect("prepare scratch cgroup root")
20625            .expect("scratch root has a cgroup.procs marker");
20626        let module_id = "reaped-module";
20627        let module = placement
20628            .module_path(module_id)
20629            .expect("create scratch module cgroup");
20630        let child = Command::new("true")
20631            .env("XDG_DATA_HOME", root.path())
20632            .env("XDG_RUNTIME_DIR", root.path())
20633            .env("XDG_CONFIG_HOME", root.path())
20634            .spawn()
20635            .expect("spawn short-lived child");
20636        let pid = child.id().expect("spawned child has pid");
20637        let mut child = SupervisedChild {
20638            registration_channel: None,
20639            registration_spawn: None,
20640            live_program_binding: None,
20641            registered_config_connection: None,
20642            admission_base: None,
20643            registration_declaration: None,
20644            _program_binding: None,
20645            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
20646            child: crate::supervise::containment::ProcessChild::plain(child, pid),
20647            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
20648            protocol: ModuleProtocol::Subc,
20649            module_id: module_id.to_string(),
20650            cgroup_placement: Some(placement),
20651            stdout_pump: None,
20652            stderr_pump: None,
20653            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
20654            spawned_at: tokio::time::Instant::now(),
20655            config_delivery: None,
20656            registration_deadline: None,
20657            registration_watch: None,
20658            spawned_at_ms: 0,
20659            spawned_from: PathBuf::from("true"),
20660            launch_spec_digest: LaunchSpecDigest([0; 32]),
20661            spawned_file_identity: None,
20662            process_start_time: None,
20663            process_identity: None,
20664            pid,
20665            roster_guard: None,
20666            #[cfg(target_os = "macos")]
20667            privacy_exec: None,
20668            spawn_failure: None,
20669            admission_kill_sent: false,
20670        };
20671
20672        child.wait().await.expect("reap short-lived child");
20673
20674        assert!(
20675            !module.exists(),
20676            "waiting for a non-carried child must remove its empty cgroup before returning"
20677        );
20678        child.finish_exit(false).await.unwrap();
20679    }
20680
20681    #[test]
20682    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
20683        let root = ScratchDir::new("supervisor-non-empty-cgroup");
20684        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20685        let placement = subc_cgroup::prepare_at(&root)
20686            .expect("prepare scratch cgroup root")
20687            .expect("scratch root has a cgroup.procs marker");
20688        let module = placement
20689            .module_path("surviving-module")
20690            .expect("create scratch module cgroup");
20691        fs::write(module.join("surviving-process"), b"still present")
20692            .expect("make scratch cgroup non-empty");
20693        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
20694
20695        remove_module_cgroup(&placement, "surviving-module");
20696
20697        let logs = crate::router::test_log::captured_logs(&logs);
20698        assert!(
20699            module.exists(),
20700            "failed removal must leave the cgroup intact"
20701        );
20702        assert!(
20703            logs.contains("could not remove module cgroup after process exit; continuing teardown")
20704                && logs.contains("surviving-module"),
20705            "best-effort removal must report the failure without returning it: {logs}"
20706        );
20707    }
20708
20709    #[test]
20710    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
20711        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
20712        let reason = SuperviseError::Spawn {
20713            program: PathBuf::from("/bin/true"),
20714            source: io::Error::from_raw_os_error(13),
20715            cgroup_path: Some(cgroup_path.clone()),
20716        }
20717        .to_string();
20718
20719        assert!(
20720            reason.contains(&cgroup_path.display().to_string()),
20721            "a pre_exec spawn failure must name the cgroup path: {reason}"
20722        );
20723    }
20724}
20725
20726#[cfg(test)]
20727mod spawn_subscriber_lag_tests {
20728    use super::*;
20729
20730    /// A subscriber whose connection stops draining is dropped once its frame
20731    /// channel fills. The client must learn that from a terminal Error frame
20732    /// after the frames already queued for it, not from a stream that simply
20733    /// goes quiet.
20734    #[tokio::test]
20735    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
20736        let feed = SpawnEventFeed::default();
20737        feed.configure_incarnation("lag-incarnation".to_string());
20738        // A one-slot connection queue that nobody reads until the emits are
20739        // done: the forwarder parks on it and the subscriber channel fills.
20740        let (tx, mut rx) = mpsc::channel(1);
20741        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
20742            .expect("subscribe");
20743        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
20744        for index in 0..emitted {
20745            feed.emit_spawned(&format!("lag-module-{index}"), None, 1000, 0);
20746            // Let the forwarder take what it can so the fill point is the
20747            // subscriber channel, not a scheduling accident.
20748            tokio::task::yield_now().await;
20749        }
20750        assert_eq!(
20751            feed.subscriber_count(),
20752            0,
20753            "the lagged subscriber must be removed"
20754        );
20755
20756        let mut data = Vec::new();
20757        let mut last = None;
20758        loop {
20759            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
20760                .await
20761                .expect("the forwarder must finish once the subscriber is dropped");
20762            let Some(outbound) = next else { break };
20763            let frame = outbound.frame;
20764            if frame.header.ty == FrameType::StreamData {
20765                assert!(last.is_none(), "no data may follow the terminal frame");
20766                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
20767                data.push(event.cursor.seq);
20768            } else {
20769                assert!(last.is_none(), "exactly one terminal frame");
20770                last = Some(frame);
20771            }
20772        }
20773        assert!(!data.is_empty(), "queued frames drain before the terminal");
20774        for pair in data.windows(2) {
20775            assert_eq!(
20776                pair[1],
20777                pair[0] + 1,
20778                "queued frames arrive dense and in order"
20779            );
20780        }
20781        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
20782        assert_eq!(terminal.header.ty, FrameType::Error);
20783        assert_eq!(terminal.header.corr, 7);
20784        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
20785        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
20786        let detail = body.detail.expect("lagged error carries detail");
20787        assert_eq!(
20788            detail["first_undelivered_cursor"]["seq"],
20789            data.last().unwrap() + 1,
20790            "the named cursor is the first event the subscriber did not receive"
20791        );
20792        assert_eq!(
20793            detail["first_undelivered_cursor"]["daemon_incarnation"],
20794            "lag-incarnation"
20795        );
20796    }
20797}
20798
20799#[cfg(test)]
20800mod terminal_history_read_concurrency_tests {
20801    use super::*;
20802    use crate::terminal_journal::read_pause;
20803    use cortexkit_test_support::ScratchDir;
20804    use std::sync::mpsc as std_mpsc;
20805
20806    fn journaled_ring(
20807        journal: &Arc<crate::terminal_journal::TerminalJournal>,
20808    ) -> Arc<Mutex<TerminalRing>> {
20809        Arc::new(Mutex::new(
20810            TerminalRing::new(TerminalRingConfig::default(), 1)
20811                .with_journal(Some(Arc::clone(journal))),
20812        ))
20813    }
20814
20815    fn crash(at_ms: u64) -> ExitReport {
20816        ExitReport {
20817            kind: ExitKind::Crash,
20818            code: Some(1),
20819            signal: None,
20820            at_ms,
20821        }
20822    }
20823
20824    /// Record an exit on another thread and report whether it finished within
20825    /// `bound`. The recorder thread is left running if it did not.
20826    fn record_within(
20827        module_id: &'static str,
20828        ring: &Arc<Mutex<TerminalRing>>,
20829        at_ms: u64,
20830        bound: Duration,
20831    ) -> bool {
20832        let ring = Arc::clone(ring);
20833        let (done, done_rx) = std_mpsc::channel();
20834        std::thread::spawn(move || {
20835            record_terminal(
20836                module_id,
20837                &ring,
20838                &SpawnEventFeed::default(),
20839                &crash(at_ms),
20840                TerminalDisposition::Restarting,
20841            );
20842            let _ = done.send(());
20843        });
20844        done_rx.recv_timeout(bound).is_ok()
20845    }
20846
20847    /// A history read in progress must not hold the journal writer (which every
20848    /// module's exit recording needs) or the module's own ring. Exits recorded
20849    /// while the read is paused complete promptly; the paused read answers as of
20850    /// the moment it started, and the next read has each exit exactly once.
20851    #[test]
20852    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
20853        let dir = ScratchDir::new("terminal-history-concurrent-read");
20854        let path = dir.join("terminals.jsonl");
20855        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
20856            path.clone(),
20857            "daemon".into(),
20858        ));
20859        let reader_ring = journaled_ring(&journal);
20860        let other_ring = journaled_ring(&journal);
20861        assert!(record_within(
20862            "reader-module",
20863            &reader_ring,
20864            10,
20865            Duration::from_secs(5)
20866        ));
20867
20868        let (started, release) = read_pause::install(&path);
20869        let reading = {
20870            let ring = Arc::clone(&reader_ring);
20871            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
20872        };
20873        started
20874            .recv_timeout(Duration::from_secs(5))
20875            .expect("the history read reached its pause");
20876
20877        let bound = Duration::from_secs(1);
20878        assert!(
20879            record_within("other-module", &other_ring, 20, bound),
20880            "another module's exit waited on a history read (journal writer held)"
20881        );
20882        assert!(
20883            record_within("reader-module", &reader_ring, 30, bound),
20884            "the read module's own exit waited on its history read (ring held)"
20885        );
20886
20887        drop(release);
20888        let paused = reading.join().unwrap();
20889        assert_eq!(
20890            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20891            vec![10],
20892            "an exit recorded after the read began lands in neither half of it"
20893        );
20894        assert_eq!(paused.journal_skipped_lines, 0);
20895        assert_eq!(paused.journal_read_errors, 0);
20896
20897        let after = durable_terminal_history_of(&reader_ring, "reader-module");
20898        assert_eq!(
20899            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20900            vec![10, 30],
20901            "the next read merges ring and journal with no duplicate"
20902        );
20903        assert_eq!(after.journal_skipped_lines, 0);
20904    }
20905}
20906
20907/// What a restart does with the exited process's output readers. These drive
20908/// the same settle functions the supervisor calls, with a reader the test
20909/// holds, so a reader that has not been scheduled by the bound is a controlled
20910/// input rather than something only a loaded machine produces.
20911#[cfg(test)]
20912mod stderr_settle_tests {
20913    use std::{
20914        future::Future,
20915        io,
20916        pin::Pin,
20917        sync::{Arc, Mutex},
20918        task::{Context, Poll},
20919        time::Duration,
20920    };
20921
20922    use tokio::{
20923        io::{AsyncRead, ReadBuf},
20924        sync::oneshot,
20925        time::Instant,
20926    };
20927
20928    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
20929    use crate::stderr_tail::{
20930        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
20931        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
20932    };
20933
20934    const BOUND: Duration = Duration::from_millis(250);
20935
20936    /// Yields `before`, then stays pending until the gate is released, then
20937    /// yields `after` and reaches EOF. The bytes after the gate were written
20938    /// by a process that has already exited; only the reader is behind.
20939    struct HeldReader {
20940        before: Option<Vec<u8>>,
20941        gate: Option<oneshot::Receiver<()>>,
20942        after: io::Cursor<Vec<u8>>,
20943    }
20944
20945    impl AsyncRead for HeldReader {
20946        fn poll_read(
20947            mut self: Pin<&mut Self>,
20948            cx: &mut Context<'_>,
20949            buf: &mut ReadBuf<'_>,
20950        ) -> Poll<io::Result<()>> {
20951            if let Some(bytes) = self.before.take() {
20952                buf.put_slice(&bytes);
20953                return Poll::Ready(Ok(()));
20954            }
20955            if let Some(gate) = self.gate.as_mut() {
20956                match Pin::new(gate).poll(cx) {
20957                    Poll::Pending => return Poll::Pending,
20958                    Poll::Ready(_) => self.gate = None,
20959                }
20960            }
20961            Pin::new(&mut self.after).poll_read(cx, buf)
20962        }
20963    }
20964
20965    struct DiscardSink;
20966
20967    impl OutputSink for DiscardSink {
20968        fn write_line(&mut self, _line: &[u8]) {}
20969    }
20970
20971    fn line(text: &str) -> TailEntry {
20972        TailEntry::Line {
20973            text: text.to_string(),
20974            truncated: false,
20975            at_ms: None,
20976        }
20977    }
20978
20979    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
20980        ring.lock().unwrap()
20981    }
20982
20983    /// Start a reader for a new process generation that delivers `before`
20984    /// immediately and `after` only once the returned sender fires (or is
20985    /// dropped).
20986    fn held_pump(
20987        ring: &Arc<Mutex<StderrRing>>,
20988        before: &str,
20989        after: &str,
20990    ) -> (StderrPump, oneshot::Sender<()>) {
20991        let generation = lock(ring).begin_process();
20992        let (release, gate) = oneshot::channel();
20993        let reader = HeldReader {
20994            before: Some(before.as_bytes().to_vec()),
20995            gate: Some(gate),
20996            after: io::Cursor::new(after.as_bytes().to_vec()),
20997        };
20998        let task = tokio::spawn(pump_stderr_to(
20999            reader,
21000            Arc::clone(ring),
21001            generation,
21002            DiscardSink,
21003        ));
21004        (StderrPump { task, generation }, release)
21005    }
21006
21007    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
21008        for _ in 0..1000 {
21009            if done(&lock(ring)) {
21010                return;
21011            }
21012            tokio::time::sleep(Duration::from_millis(1)).await;
21013        }
21014        panic!(
21015            "ring never reached the expected state: {:?}",
21016            lock(ring).snapshot(None, None)
21017        );
21018    }
21019
21020    #[tokio::test(start_paused = true)]
21021    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
21022        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21023        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
21024
21025        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
21026        let before_release = lock(&ring).snapshot(None, None);
21027        assert!(
21028            matches!(before_release.capture, CaptureState::Incomplete { .. }),
21029            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
21030        );
21031
21032        // The restart: the next process starts and writes before the old
21033        // reader catches up.
21034        let next = lock(&ring).begin_process();
21035        lock(&ring).push_line_from(next, "next process booting");
21036        release.send(()).unwrap();
21037        wait_until(&ring, |ring| {
21038            ring.snapshot(None, None).capture == CaptureState::Captured
21039        })
21040        .await;
21041
21042        assert_eq!(
21043            untimed(lock(&ring).snapshot(None, None).entries),
21044            vec![
21045                line("booting"),
21046                line("config error: missing storage"),
21047                TailEntry::ProcessStart,
21048                line("next process booting"),
21049            ],
21050            "the crash's last line must survive a slow reader and stay in the crashed process's section"
21051        );
21052    }
21053
21054    #[tokio::test(start_paused = true)]
21055    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
21056    ) {
21057        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21058        // `_held` is never fired: a descendant keeps the pipe open for the
21059        // whole test.
21060        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
21061
21062        let started = Instant::now();
21063        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
21064        assert_eq!(
21065            started.elapsed(),
21066            BOUND,
21067            "the restart must wait exactly the bound for a pipe that stays open, no longer"
21068        );
21069
21070        let next = lock(&ring).begin_process();
21071        lock(&ring).push_line_from(next, "next process booting");
21072        tokio::time::sleep(Duration::from_secs(60)).await;
21073
21074        let snapshot = lock(&ring).snapshot(None, None);
21075        match &snapshot.capture {
21076            CaptureState::Incomplete { reason } => assert!(
21077                reason.contains("had not reached EOF") && reason.contains("250ms"),
21078                "the reason must say what is missing and after how long: {reason}"
21079            ),
21080            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
21081        }
21082        assert_eq!(
21083            untimed(snapshot.entries),
21084            vec![
21085                line("parent exiting"),
21086                TailEntry::ProcessStart,
21087                line("next process booting"),
21088            ]
21089        );
21090    }
21091
21092    #[tokio::test(start_paused = true)]
21093    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
21094        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21095        let (pump, release) = held_pump(&ring, "one\n", "two\n");
21096        release.send(()).unwrap();
21097
21098        settle_stderr_pump("clean", &ring, pump, BOUND).await;
21099
21100        let snapshot = lock(&ring).snapshot(None, None);
21101        assert_eq!(snapshot.capture, CaptureState::Captured);
21102        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
21103    }
21104
21105    #[tokio::test(start_paused = true)]
21106    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
21107        const BURST: usize = 150;
21108        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
21109        let path = capture.join("two-pipe.stderr.log");
21110        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
21111        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21112        let generation = lock(&ring).begin_process();
21113        let stdout = (0..BURST)
21114            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
21115            .collect::<String>();
21116        let stderr = (0..BURST)
21117            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
21118            .collect::<String>();
21119        // Model a child that has written both bursts and exited. Hold the stdout
21120        // reader after one line and a partial prefix, as if pipe readiness or
21121        // capture-file writes had delayed it. The final line also needs EOF to
21122        // flush: neither complete nor partial buffered output may be discarded.
21123        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
21124        let (release, gate) = oneshot::channel();
21125        let reader = HeldReader {
21126            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
21127            gate: Some(gate),
21128            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
21129        };
21130        let (finished, completion) = oneshot::channel();
21131        let stdout_sink = sink.clone();
21132        let pump = tokio::spawn(async move {
21133            pump_stdout_to(reader, stdout_sink).await;
21134            finished.send(()).unwrap();
21135        });
21136        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
21137
21138        let started = Instant::now();
21139        settle_stdout_pump("two-pipe", pump, BOUND).await;
21140        assert_eq!(
21141            started.elapsed(),
21142            BOUND,
21143            "a late reader must not hold up restart"
21144        );
21145        assert_eq!(
21146            std::fs::read_to_string(&path).unwrap().lines().count(),
21147            BURST + 1
21148        );
21149
21150        release
21151            .send(())
21152            .expect("stdout reader must survive the exit drain bound");
21153        tokio::time::timeout(Duration::from_secs(1), completion)
21154            .await
21155            .expect("released stdout reader must reach EOF")
21156            .expect("stdout capture must finish at EOF, not be cancelled");
21157        let contents = std::fs::read_to_string(&path).unwrap();
21158        let mut lines = contents
21159            .lines()
21160            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
21161            .collect::<Vec<_>>();
21162        let mut expected = (0..BURST)
21163            .flat_map(|i| {
21164                [
21165                    format!("out-{i:04}-{}", "o".repeat(64)),
21166                    format!("err-{i:04}-{}", "e".repeat(64)),
21167                ]
21168            })
21169            .collect::<Vec<_>>();
21170        lines.sort_unstable();
21171        expected.sort_unstable();
21172        assert_eq!(
21173            lines, expected,
21174            "both pipes must retain every complete line exactly once"
21175        );
21176    }
21177}
21178
21179/// Containment of a module's process tree (issue #109).
21180///
21181/// The behaviour these defend against is a module helper surviving its module:
21182/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
21183/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
21184/// compounds it.
21185///
21186/// They run against the SUPERVISOR rather than the job-object crate because the
21187/// claim is about teardown: a crate-level test proves a job can reap a tree, not
21188/// that the daemon's drain path reaches it.
21189///
21190/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
21191/// lane there is a separate containment path with its own tests.
21192#[cfg(all(test, windows))]
21193mod job_containment_tests {
21194    use super::*;
21195    use cortexkit_test_support::ScratchDir;
21196    use std::{
21197        path::{Path, PathBuf},
21198        sync::{Arc, Mutex},
21199        time::{Duration, Instant},
21200    };
21201
21202    /// The stub, expected beside this test executable.
21203    ///
21204    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
21205    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
21206    /// failure then reads as a broken test rather than an unbuilt dependency.
21207    fn stub_path() -> PathBuf {
21208        let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
21209        assert!(
21210            path.exists(),
21211            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
21212             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
21213            path.display()
21214        );
21215        path
21216    }
21217
21218    /// Poll for the grandchild pid the stub records, and parse it.
21219    fn read_grandchild_pid(path: &Path) -> u32 {
21220        let deadline = Instant::now() + Duration::from_secs(10);
21221        loop {
21222            if let Ok(contents) = std::fs::read_to_string(path) {
21223                if let Ok(pid) = contents.trim().parse() {
21224                    return pid;
21225                }
21226            }
21227            assert!(
21228                Instant::now() < deadline,
21229                "the stub never recorded a grandchild pid at {}",
21230                path.display()
21231            );
21232            std::thread::sleep(Duration::from_millis(10));
21233        }
21234    }
21235
21236    /// Everything one fixture run needs, so the two tests below differ in exactly
21237    /// one place: whether the child is contained.
21238    struct Fixture {
21239        _dir: ScratchDir,
21240        _roster: ChildRoster,
21241        module_id: String,
21242        grandchild: u32,
21243        child: Option<SupervisedChild>,
21244        registry: Arc<Registry>,
21245        snapshot: Arc<Mutex<SupervisorSnapshot>>,
21246        terminal_ring: Arc<Mutex<TerminalRing>>,
21247        spawn_events: SpawnEventFeed,
21248    }
21249
21250    fn fixture(label: &str, module_id: &str) -> Fixture {
21251        fixture_with_carry(label, module_id, false)
21252    }
21253
21254    fn fixture_with_carry(label: &str, module_id: &str, carry_containment: bool) -> Fixture {
21255        let dir = ScratchDir::new(label);
21256        let pid_file = dir.join("grandchild.pid");
21257        let supervisor = Supervisor::new_for_test(
21258            Arc::new(Registry::default()),
21259            RestartPolicy::new(3, Duration::ZERO),
21260        );
21261        let runtime = supervisor.runtime_config();
21262        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21263        let spec = ModuleSpec {
21264            script: None,
21265            module_id: module_id.to_string(),
21266            program: stub_path(),
21267            // Zero args deliberately: a `--subc` argument would make the stub dial
21268            // a daemon that is not there, and the failure would land in the same
21269            // stderr ring this fixture exists to keep quiet.
21270            args: Vec::new(),
21271            env: vec![
21272                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
21273                (
21274                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
21275                    pid_file.display().to_string(),
21276                ),
21277            ],
21278            reserved: false,
21279            reserved_prefixes: Vec::new(),
21280            protocol: ModuleProtocol::Subc,
21281            overlap: Default::default(),
21282            carry_containment,
21283        };
21284        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
21285            .expect("spawn the supervised fixture");
21286        let grandchild = read_grandchild_pid(&pid_file);
21287        Fixture {
21288            _dir: dir,
21289            _roster: runtime.child_roster.clone(),
21290            module_id: module_id.to_string(),
21291            grandchild,
21292            child: Some(child),
21293            registry: Arc::new(Registry::default()),
21294            snapshot,
21295            terminal_ring: Arc::clone(&runtime.terminal_ring),
21296            spawn_events: SpawnEventFeed::default(),
21297        }
21298    }
21299
21300    async fn assert_admission_kill_reaps_grandchild(carry: bool) {
21301        let label = if carry {
21302            "admission-kill-carried-tree"
21303        } else {
21304            "admission-kill-plain-tree"
21305        };
21306        let mut fixture = fixture_with_carry(label, label, carry);
21307        let descendant = subc_os::Process::open(fixture.grandchild)
21308            .unwrap()
21309            .expect("the stub descendant must be alive before admission kill");
21310        assert!(descendant.observe().is_some());
21311        let child = fixture.child.as_mut().unwrap();
21312        assert_eq!(child.child.container().is_some(), carry);
21313        assert_eq!(child.job.is_some(), !carry);
21314
21315        child.start_admission_kill().unwrap();
21316        assert!(child.admission_kill_sent);
21317        let status = timeout(Duration::from_secs(10), child.wait())
21318            .await
21319            .unwrap()
21320            .unwrap();
21321        assert_eq!(
21322            status.code(),
21323            Some(admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE),
21324            "containment teardown must preserve the leader's distinctive status"
21325        );
21326        // Keep the child and its job/container alive: later exit finalization or
21327        // handle drop must not be what makes the descendant-death claim pass.
21328        assert!(
21329            descendant.wait_for_exit(Duration::from_secs(10)).unwrap(),
21330            "admission kill left a descendant running before exit finalization"
21331        );
21332        assert_eq!(child.child.container().is_some(), carry);
21333        assert_eq!(child.job.is_some(), !carry);
21334        child.finish_exit(false).await.unwrap();
21335    }
21336
21337    #[tokio::test]
21338    async fn admission_kill_reaps_grandchild_with_plain_job() {
21339        assert_admission_kill_reaps_grandchild(false).await;
21340    }
21341
21342    #[tokio::test]
21343    async fn admission_kill_reaps_grandchild_with_carried_job() {
21344        assert_admission_kill_reaps_grandchild(true).await;
21345    }
21346
21347    #[tokio::test]
21348    async fn admission_kill_retires_registration_before_wait() {
21349        let mut fixture = fixture("admission-registration", "admission-registration");
21350        let handle = SupervisorHandle::new();
21351        let spawn =
21352            RegistrationSpawn::new(&handle, "admission-registration", SpawnRole::Plain, None);
21353        spawn.ready();
21354        let child = fixture.child.as_mut().unwrap();
21355        child.registration_spawn = Some(spawn.clone());
21356        child.start_admission_kill().unwrap();
21357        assert!(
21358            !spawn.launch_ready(),
21359            "admission kill must retire before its caller can await the child"
21360        );
21361        timeout(Duration::from_secs(10), child.wait())
21362            .await
21363            .unwrap()
21364            .unwrap();
21365        child.finish_exit(false).await.unwrap();
21366    }
21367
21368    impl Fixture {
21369        /// Drain through the supervisor's own teardown path.
21370        async fn drain(&mut self) {
21371            let child = self
21372                .child
21373                .take()
21374                .expect("the fixture child is still present");
21375            drain_child_to_state(
21376                &self.module_id,
21377                ModuleProtocol::Subc,
21378                // No forwarding table in this fixture, so nothing reaches the
21379                // child over a connection.
21380                StopNotice::NotSent,
21381                &self.registry,
21382                None,
21383                &self.snapshot,
21384                &self.terminal_ring,
21385                &self.spawn_events,
21386                child,
21387                Duration::from_millis(500),
21388                ModuleState::Stopped,
21389                Some(false),
21390                &FailureContext::operator(),
21391            )
21392            .await
21393            .expect("drain the supervised fixture");
21394        }
21395    }
21396
21397    /// Teardown reaps the grandchild, not merely the direct child.
21398    ///
21399    /// This is the assertion the change exists for. Before containment the
21400    /// grandchild survived: it is a separate process, and `start_kill` is
21401    /// `TerminateProcess` scoped to one pid.
21402    #[tokio::test]
21403    async fn teardown_reaps_the_grandchild() {
21404        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
21405        let grandchild = fixture.grandchild;
21406
21407        assert!(
21408            subc_jobobject::process_exists(grandchild),
21409            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
21410        );
21411
21412        fixture.drain().await;
21413
21414        assert!(
21415            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21416            "grandchild {grandchild} outlived module teardown: the tree was not contained"
21417        );
21418    }
21419
21420    /// The mutation control: with containment withheld, the grandchild survives
21421    /// the same kill.
21422    ///
21423    /// This is the defect reproduction from #109 — a direct-child kill reaches
21424    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
21425    /// supervisor because `spawn_and_mark_running` now always contains on
21426    /// Windows, which is the point: there is no longer a path that spawns
21427    /// uncontained, so the control has to construct one.
21428    ///
21429    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
21430    /// grandchild ever dies here, that test is passing for a reason unrelated to
21431    /// the job object and the containment claim is unproven.
21432    #[test]
21433    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
21434        let dir = ScratchDir::new("teardown-uncontained");
21435        let pid_file = dir.join("grandchild.pid");
21436        let mut child = std::process::Command::new(stub_path())
21437            .env("FAKE_AFT_NEVER_CONNECT", "1")
21438            .env(
21439                "FAKE_AFT_GRANDCHILD_PID_FILE",
21440                pid_file.display().to_string(),
21441            )
21442            .stdin(std::process::Stdio::null())
21443            .stdout(std::process::Stdio::null())
21444            .stderr(std::process::Stdio::null())
21445            .spawn()
21446            .expect("spawn the uncontained fixture");
21447        let grandchild = read_grandchild_pid(&pid_file);
21448
21449        // Exactly what the pre-fix teardown did: kill the direct child.
21450        child.kill().expect("kill the direct child");
21451        let _ = child.wait();
21452
21453        assert!(
21454            subc_jobobject::process_exists(grandchild),
21455            "grandchild {grandchild} died with the direct child, so this control no longer \
21456             distinguishes contained from uncontained teardown and the regression test is \
21457             passing vacuously"
21458        );
21459
21460        // The orphan this control demonstrates is the leak the fix prevents, so
21461        // the control must not leave one behind.
21462        kill_tree(grandchild);
21463    }
21464
21465    /// Crash durability: closing the containment handle reaps the tree with no
21466    /// teardown code running at all.
21467    ///
21468    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
21469    /// call anything — and it is why containment is a kernel property of the
21470    /// handle rather than a step in the drain. Discovered by getting the
21471    /// mutation control wrong: clearing `job` to "disable" containment instead
21472    /// killed the tree, which is the guarantee, not a mistake.
21473    #[tokio::test]
21474    async fn dropping_containment_reaps_the_grandchild() {
21475        let mut fixture = fixture("drop-containment", "tree-drop");
21476        let grandchild = fixture.grandchild;
21477
21478        assert!(subc_jobobject::process_exists(grandchild));
21479
21480        // No `drain` call, no kill: dropping the handle is the entire mechanism.
21481        fixture.child.as_mut().expect("child present").job = None;
21482
21483        assert!(
21484            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21485            "grandchild {grandchild} survived the containment handle closing, so a daemon \
21486             crash would leave the tree behind"
21487        );
21488    }
21489
21490    /// Kill a pid and its tree, then confirm it is gone.
21491    fn kill_tree(pid: u32) {
21492        let _ = std::process::Command::new("taskkill.exe")
21493            .args(["/PID", &pid.to_string(), "/T", "/F"])
21494            .stdin(std::process::Stdio::null())
21495            .stdout(std::process::Stdio::null())
21496            .stderr(std::process::Stdio::null())
21497            .status();
21498        assert!(
21499            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
21500            "could not clean up grandchild {pid}"
21501        );
21502    }
21503}
21504
21505#[cfg(test)]
21506mod privacy_trampoline_configuration_tests {
21507    #[tokio::test]
21508    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21509    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
21510        #[cfg(target_os = "macos")]
21511        {
21512            let supervisor = super::Supervisor::new(
21513                std::sync::Arc::new(crate::Registry::default()),
21514                super::RestartPolicy::default(),
21515            );
21516            let error = supervisor.spawn(spec()).unwrap_err();
21517            assert!(
21518                error
21519                    .to_string()
21520                    .contains("no privacy trampoline configured"),
21521                "{error}"
21522            );
21523        }
21524    }
21525
21526    #[tokio::test]
21527    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21528    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
21529        #[cfg(target_os = "macos")]
21530        {
21531            let supervisor = super::Supervisor::new(
21532                std::sync::Arc::new(crate::Registry::default()),
21533                super::RestartPolicy::default(),
21534            )
21535            .with_privacy_trampoline(std::env::current_exe().unwrap());
21536            let error = supervisor.spawn(spec()).unwrap_err();
21537            assert!(
21538                error
21539                    .to_string()
21540                    .contains("binary does not implement the privacy trampoline protocol"),
21541                "{error}"
21542            );
21543        }
21544    }
21545
21546    #[cfg(target_os = "macos")]
21547    fn spec() -> super::ModuleSpec {
21548        super::ModuleSpec {
21549            module_id: "privacy-configuration".into(),
21550            program: "/bin/sleep".into(),
21551            args: vec!["30".into()],
21552            env: vec![],
21553            reserved: false,
21554            reserved_prefixes: vec![],
21555            protocol: subc_control::ModuleProtocol::None,
21556            overlap: super::ModuleOverlap::Exclusive,
21557            carry_containment: false,
21558            script: None,
21559        }
21560    }
21561}
21562
21563#[cfg(test)]
21564mod privacy_exec_boundary_tests {
21565    #[cfg(target_os = "macos")]
21566    use super::*;
21567    #[cfg(target_os = "macos")]
21568    use std::{
21569        io::{Read, Write},
21570        net::{TcpListener, TcpStream},
21571    };
21572
21573    /// Unit-test-only pause at the actual early image read, not at a later
21574    /// status read. Production supervisors never inspect this environment key.
21575    #[cfg(target_os = "macos")]
21576    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
21577        if let Some((_, path)) = spec
21578            .env
21579            .iter()
21580            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
21581        {
21582            let mut barrier = TcpStream::connect(path).unwrap();
21583            barrier
21584                .set_read_timeout(Some(Duration::from_secs(30)))
21585                .unwrap();
21586            barrier.write_all(&pid.to_ne_bytes()).unwrap();
21587            let mut release = [0];
21588            barrier.read_exact(&mut release).unwrap();
21589            assert_eq!(&release, b"X");
21590        }
21591    }
21592
21593    #[cfg(target_os = "macos")]
21594    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
21595        ModuleSpec {
21596            script: None,
21597            module_id: "privacy-boundary".into(),
21598            program: program.into(),
21599            args: args.iter().map(|arg| (*arg).into()).collect(),
21600            env: vec![],
21601            reserved: false,
21602            reserved_prefixes: vec![],
21603            protocol: ModuleProtocol::None,
21604            overlap: ModuleOverlap::Exclusive,
21605            carry_containment: false,
21606        }
21607    }
21608
21609    #[cfg(target_os = "macos")]
21610    async fn accept(listener: TcpListener) -> TcpStream {
21611        // Socket readiness, not elapsed time, establishes both pause points.
21612        let listener = tokio::net::TcpListener::from_std({
21613            listener.set_nonblocking(true).unwrap();
21614            listener
21615        })
21616        .unwrap();
21617        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
21618            .await
21619            .unwrap()
21620            .unwrap();
21621        let stream = stream.into_std().unwrap();
21622        stream.set_nonblocking(false).unwrap();
21623        stream
21624            .set_read_timeout(Some(Duration::from_secs(30)))
21625            .unwrap();
21626        stream
21627    }
21628
21629    #[tokio::test]
21630    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21631    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
21632        #[cfg(target_os = "macos")]
21633        {
21634            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
21635            // Loopback sockets also work when the replay adapter's TMPDIR is
21636            // longer than Darwin's Unix-domain socket path limit.
21637            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21638            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21639            let record = root.join("live-children.json");
21640            let supervisor =
21641                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21642                    .with_live_children_record(&record);
21643            let runtime = supervisor.runtime_config();
21644            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21645            let mut spec = spec("/bin/sleep", &["30"]);
21646            spec.env = vec![
21647                (
21648                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21649                    exec_listener.local_addr().unwrap().to_string(),
21650                ),
21651                (
21652                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
21653                    sample_listener.local_addr().unwrap().to_string(),
21654                ),
21655            ];
21656            let spawn = tokio::task::spawn_blocking(move || {
21657                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
21658            });
21659            let mut sample = accept(sample_listener).await;
21660            let mut pid = [0; 4];
21661            sample.read_exact(&mut pid).unwrap();
21662            let pid = u32::from_ne_bytes(pid);
21663            let mut exec = accept(exec_listener).await;
21664            let mut ready = [0];
21665            exec.read_exact(&mut ready).unwrap();
21666            assert_eq!(&ready, b"R");
21667            // The early read is guaranteed to see a real, non-null trampoline
21668            // image: the fixture has reached its barrier and cannot exec yet.
21669            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
21670            assert_eq!(
21671                observe_spawned_image(pid).unwrap().executable,
21672                Some(trampoline)
21673            );
21674            sample.write_all(b"X").unwrap();
21675            let mut child = spawn.await.unwrap();
21676            let early = crate::live_children::read_record(&record).unwrap();
21677            assert_eq!(early.len(), 1);
21678            assert_eq!(early[0].pid, pid);
21679            assert_eq!(
21680                early[0].executable, None,
21681                "unconfirmed trampoline image entered the roster"
21682            );
21683            assert!(child.report_ready.get().is_none());
21684            // The barrier's duration is unrelated to the production five-second
21685            // exec budget. Start the test's confirmation budget upon release.
21686            child.privacy_exec.as_mut().unwrap().deadline =
21687                tokio::time::Instant::now() + Duration::from_secs(30);
21688            exec.write_all(b"X").unwrap();
21689            child.confirm_privacy_exec().await;
21690            assert_eq!(child.spawn_failure, None);
21691            assert!(child.report_ready.get().is_some());
21692            let confirmed = crate::live_children::read_record(&record).unwrap();
21693            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
21694            assert_ne!(module, trampoline);
21695            assert_eq!(confirmed[0].executable, Some(module.into()));
21696            child.start_kill().unwrap();
21697            child.wait().await.unwrap();
21698            child.release_roster();
21699        }
21700    }
21701
21702    #[tokio::test]
21703    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21704    async fn macos_unstamped_reexec_records_the_observed_native_image() {
21705        #[cfg(target_os = "macos")]
21706        {
21707            let root = cortexkit_test_support::ScratchDir::new("privacy-reexec");
21708            let record = root.join("live-children.json");
21709            let supervisor =
21710                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21711                    .with_live_children_record(&record);
21712            let runtime = supervisor.runtime_config();
21713            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21714            let fixture = crate::config_gate::program::native_fixture_for_test();
21715            let mut spec = spec(
21716                "/bin/sh",
21717                &["-c", "exec \"$1\" 600", "reexec", fixture.to_str().unwrap()],
21718            );
21719            spec.env
21720                .push(("SUBC_TEST_PRIVACY_IMAGE_DELAY_MS".into(), "5".into()));
21721            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21722            let native_image = subc_os::file_identity(&fixture).unwrap();
21723            // Native image installation, not shell startup speed, establishes
21724            // that this launch has crossed the re-exec boundary before sampling.
21725            let reexec = tokio::time::timeout(Duration::from_secs(5), async {
21726                while observe_spawned_image(child.pid).and_then(|image| image.executable)
21727                    != Some(native_image)
21728                {
21729                    tokio::time::sleep(Duration::from_millis(1)).await;
21730                }
21731            })
21732            .await;
21733            child.confirm_privacy_exec().await;
21734            let failure = child.spawn_failure.clone();
21735            let ready = child.report_ready.get().is_some();
21736            let confirmed = crate::live_children::read_record(&record).unwrap();
21737            child.start_kill().unwrap();
21738            tokio::time::timeout(Duration::from_secs(5), child.wait())
21739                .await
21740                .unwrap()
21741                .unwrap();
21742            child.release_roster();
21743
21744            reexec.expect("the shell must re-exec the native fixture");
21745            assert_eq!(
21746                failure, None,
21747                "an unstamped program may re-exec before observation"
21748            );
21749            assert!(ready);
21750            assert_eq!(confirmed.len(), 1);
21751            assert_eq!(confirmed[0].executable, Some(native_image.into()));
21752        }
21753    }
21754
21755    #[tokio::test]
21756    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21757    async fn macos_unstamped_ack_without_exec_still_refuses_the_trampoline() {
21758        #[cfg(target_os = "macos")]
21759        {
21760            let listener = TcpListener::bind("127.0.0.1:0").unwrap();
21761            let supervisor =
21762                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
21763            let runtime = supervisor.runtime_config();
21764            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21765            let fixture = crate::config_gate::program::native_fixture_for_test();
21766            let mut spec = spec(fixture.to_str().unwrap(), &["600"]);
21767            spec.env.push((
21768                "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21769                listener.local_addr().unwrap().to_string(),
21770            ));
21771            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21772            let mut exec = accept(listener).await;
21773            let mut ready = [0];
21774            exec.read_exact(&mut ready).unwrap();
21775            assert_eq!(&ready, b"R");
21776            assert_eq!(
21777                observe_spawned_image(child.pid).unwrap().executable,
21778                subc_os::file_identity(&test_privacy_trampoline())
21779            );
21780            // Simulate a broken trampoline that acknowledges exec but keeps
21781            // running its own image, held at the fixture's pre-exec barrier.
21782            let (reader, writer) = std::io::pipe().unwrap();
21783            drop(writer);
21784            child.privacy_exec.as_mut().unwrap().reader =
21785                tokio::io::unix::AsyncFd::new(reader).unwrap();
21786            child.confirm_privacy_exec().await;
21787            let failure = child.spawn_failure.clone();
21788            let ready = child.report_ready.get().is_some();
21789            child.start_kill().unwrap();
21790            tokio::time::timeout(Duration::from_secs(5), child.wait())
21791                .await
21792                .unwrap()
21793                .unwrap();
21794            child.release_roster();
21795            assert_eq!(
21796                failure.as_deref(),
21797                Some("privacy identity trampoline did not exec the module")
21798            );
21799            assert!(!ready);
21800        }
21801    }
21802
21803    #[tokio::test]
21804    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21805    async fn macos_resumed_121_is_a_module_exit_not_a_trampoline_refusal() {
21806        #[cfg(target_os = "macos")]
21807        {
21808            let registry = Arc::new(Registry::default());
21809            let policy = RestartPolicy::new(0, Duration::ZERO);
21810            let fixture = crate::config_gate::program::native_fixture_for_test();
21811            let spec = spec(fixture.to_str().unwrap(), &["exit121"]);
21812            let scratch = cortexkit_test_support::ScratchDir::new("resumed-exit");
21813            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy)
21814                .with_config_runtime(suspended_spawn_tests::gated_owner(&scratch, &spec, true));
21815            let runtime = supervisor.runtime_config();
21816            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21817            // A module cannot exit before suspended admission. After resume,
21818            // retain the real exit status even when it resembles a refusal.
21819            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21820            child.confirm_privacy_exec().await;
21821            assert_eq!(child.spawn_failure, None);
21822            assert!(child.report_ready.get().is_some());
21823            let pid = child.pid;
21824            tokio::task::spawn_blocking(move || {
21825                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
21826            })
21827            .await
21828            .unwrap()
21829            .unwrap();
21830            let status = child.wait().await.unwrap();
21831            assert_eq!(status.code(), Some(121));
21832            assert!(child.privacy_exec.is_none());
21833            let report = classify_reaped_child_exit(&snapshot, &child, &status);
21834            on_child_exit(
21835                &spec,
21836                policy,
21837                &registry,
21838                &snapshot,
21839                &runtime.terminal_ring,
21840                &runtime.spawn_events,
21841                &runtime.child_roster,
21842                report,
21843                true,
21844            )
21845            .await;
21846            let state = lock_snapshot(&snapshot).unwrap();
21847            assert_eq!(state.state, ModuleState::Failed);
21848            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
21849            assert_eq!(state.reported_pid(), None);
21850            drop(state);
21851            let history = runtime.terminal_ring.lock().unwrap().snapshot();
21852            assert_eq!(history.entries.len(), 1);
21853            let terminal = &history.entries[0];
21854            assert_eq!(terminal.exit_code, Some(121));
21855            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
21856            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
21857            assert_eq!(
21858                terminal.disposition_detail.as_deref(),
21859                Some(policy.budget_exhausted_detail().as_str()),
21860                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
21861            );
21862            assert_eq!(child.spawn_failure, None);
21863            child.release_roster();
21864        }
21865    }
21866}
21867
21868#[cfg(all(test, target_os = "macos"))]
21869#[path = "supervise/suspended_spawn_tests.rs"]
21870mod suspended_spawn_tests;
21871
21872/// The daemon's real spawn path hands a subc-wire child its launch nonce on
21873/// descriptor 3, without an environment copy. The shell records the nonce
21874/// and its environment after exec so these tests observe the real handover.
21875#[cfg(all(test, unix))]
21876mod launch_nonce_descriptor_tests {
21877    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
21878    use crate::stderr_tail::{StderrRing, StderrTailConfig};
21879    use cortexkit_test_support::ScratchDir;
21880    use std::{
21881        path::PathBuf,
21882        sync::{Arc, Mutex},
21883        time::{Duration, Instant},
21884    };
21885
21886    async fn probe(role: super::SpawnRole) {
21887        let scratch = ScratchDir::new("launch-nonce-descriptor");
21888        let fd_copy = scratch.join("from-descriptor");
21889        let env_copy = scratch.join("environment");
21890        let script = format!(
21891            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
21892            fd = fd_copy.display(), env = env_copy.display(),
21893        );
21894        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
21895        let spec = ModuleSpec {
21896            script: None,
21897            module_id: "nonce-descriptor-probe".to_string(),
21898            program: PathBuf::from("/bin/sh"),
21899            args: vec!["-c".to_string(), script],
21900            env: vec![
21901                xdg("XDG_DATA_HOME"),
21902                xdg("XDG_RUNTIME_DIR"),
21903                xdg("XDG_CONFIG_HOME"),
21904                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
21905            ],
21906            reserved: true,
21907            reserved_prefixes: Vec::new(),
21908            protocol: ModuleProtocol::Subc,
21909            overlap: Default::default(),
21910            carry_containment: false,
21911        };
21912        let handle = SupervisorHandle::new();
21913        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21914        let roster = ChildRoster::default();
21915        #[cfg(target_os = "macos")]
21916        {
21917            let path = super::test_privacy_trampoline();
21918            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
21919        }
21920        let child = super::spawn_child_in_slot(
21921            &spec,
21922            None,
21923            Some(&handle),
21924            &ring,
21925            None,
21926            &roster,
21927            None,
21928            #[cfg(target_os = "linux")]
21929            None,
21930            role,
21931            matches!(role, super::SpawnRole::SwapCandidate),
21932            super::DEFAULT_REGISTER_TIMEOUT,
21933        )
21934        .expect("spawn probe");
21935        let deadline = Instant::now() + Duration::from_secs(10);
21936        while !(fd_copy.exists() && env_copy.exists()) {
21937            assert!(Instant::now() < deadline, "probe never wrote its copies");
21938            tokio::time::sleep(Duration::from_millis(20)).await;
21939        }
21940        let nonce = std::fs::read_to_string(fd_copy).unwrap();
21941        assert!(!nonce.is_empty());
21942        let environment = std::fs::read_to_string(env_copy).unwrap();
21943        assert!(environment
21944            .lines()
21945            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
21946        let copy = environment
21947            .lines()
21948            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
21949        assert_eq!(
21950            copy, None,
21951            "Unix children must never receive the environment nonce"
21952        );
21953        if matches!(role, super::SpawnRole::Plain) {
21954            assert_eq!(
21955                handle.spawn_nonce(&spec.module_id).as_deref(),
21956                Some(nonce.as_str())
21957            );
21958        }
21959        drop(child);
21960    }
21961
21962    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
21963    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
21964        probe(super::SpawnRole::Plain).await;
21965    }
21966
21967    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
21968    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
21969        probe(super::SpawnRole::SwapCandidate).await;
21970    }
21971}
21972
21973#[cfg(all(test, target_os = "linux"))]
21974#[path = "supervise/cgroup_reap_tests.rs"]
21975mod cgroup_reap_tests;
21976
21977#[cfg(all(test, target_os = "linux"))]
21978mod cgroup_containment_tests {
21979    use super::*;
21980    use cortexkit_test_support::ScratchDir;
21981
21982    #[tokio::test]
21983    async fn linux_subtree_memory_covers_module_children() {
21984        let dir = ScratchDir::new("subtree-memory");
21985        let root = PathBuf::from(format!(
21986            "/sys/fs/cgroup/subc-memory-test-{}-{}",
21987            std::process::id(),
21988            unix_ms_now()
21989        ));
21990        if let Err(error) = std::fs::create_dir(&root) {
21991            assert!(
21992                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
21993                "required cgroup test cannot execute: {error}"
21994            );
21995            eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
21996            return;
21997        }
21998        let placement = subc_cgroup::prepare_at(&root)
21999            .unwrap()
22000            .expect("delegated cgroup");
22001        let supervisor = Supervisor::new_for_test(
22002            Arc::new(Registry::default()),
22003            RestartPolicy::new(3, Duration::ZERO),
22004        )
22005        .with_cgroup_placement(Some(placement));
22006        let ready = dir.join("ready");
22007        let module = supervisor
22008            .spawn(ModuleSpec {
22009                script: None,
22010                module_id: "memory-tree".into(),
22011                program: PathBuf::from("/bin/sh"),
22012                args: vec![
22013                    "-c".into(),
22014                    "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
22015                    "fixture".into(),
22016                    ready.display().to_string(),
22017                ],
22018                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
22019                    .into_iter()
22020                    .map(|key| (key.into(), dir.display().to_string()))
22021                    .collect(),
22022                reserved: false,
22023                reserved_prefixes: Vec::new(),
22024                protocol: ModuleProtocol::None,
22025                overlap: Default::default(),
22026                carry_containment: false,
22027            })
22028            .unwrap();
22029        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
22030        while !ready.exists() {
22031            assert!(
22032                tokio::time::Instant::now() < deadline,
22033                "children never started"
22034            );
22035            tokio::time::sleep(Duration::from_millis(10)).await;
22036        }
22037        let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
22038        let reading = &readings[0];
22039        assert!(
22040            reading.process_count.is_some_and(|count| count >= 4),
22041            "{reading:?}"
22042        );
22043        assert!(
22044            reading.memory_bytes.is_some_and(|bytes| bytes > 0),
22045            "{reading:?}"
22046        );
22047        module.stop().await.unwrap();
22048        drop(module);
22049        std::fs::remove_dir(root.join("subc-modules")).unwrap();
22050        std::fs::remove_dir(&root).unwrap();
22051        eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
22052    }
22053
22054    fn running(pid: u32) -> bool {
22055        // An orphan can remain a zombie until the container init reaps it.
22056        std::fs::read_to_string(format!("/proc/{pid}/stat"))
22057            .ok()
22058            .and_then(|stat| {
22059                stat.rsplit_once(") ")
22060                    .map(|(_, rest)| rest.starts_with('Z'))
22061            })
22062            .is_some_and(|zombie| !zombie)
22063    }
22064
22065    #[tokio::test]
22066    async fn linux_teardown_reaps_the_grandchild() {
22067        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
22068    }
22069
22070    #[tokio::test]
22071    async fn linux_shutdown_straggler_reaps_the_grandchild() {
22072        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
22073    }
22074
22075    async fn teardown_tree(test_name: &str, shutdown: bool) {
22076        let dir = ScratchDir::new(test_name);
22077        let root = PathBuf::from(format!(
22078            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
22079            std::process::id(),
22080            unix_ms_now()
22081        ));
22082        if let Err(error) = std::fs::create_dir(&root) {
22083            assert!(
22084                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
22085                "required cgroup test cannot execute: {error}"
22086            );
22087            eprintln!(
22088                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
22089                root.display()
22090            );
22091            return;
22092        }
22093        let placement = subc_cgroup::prepare_at(&root)
22094            .expect("prepare isolated kernel cgroup")
22095            .expect("isolated cgroup is delegated");
22096        let module_id = "tree-teardown";
22097        let module = placement
22098            .module_path(module_id)
22099            .expect("create isolated module cgroup");
22100        if !module.join("cgroup.kill").exists() {
22101            std::fs::remove_dir(&module).unwrap();
22102            std::fs::remove_dir(root.join("subc-modules")).unwrap();
22103            std::fs::remove_dir(&root).unwrap();
22104            assert!(
22105                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
22106                "required cgroup.kill interface unavailable"
22107            );
22108            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
22109            return;
22110        }
22111        let supervisor = Supervisor::new_for_test(
22112            Arc::new(Registry::default()),
22113            RestartPolicy::new(3, Duration::ZERO),
22114        )
22115        .with_cgroup_placement(Some(placement));
22116        let mut runtime = supervisor.runtime_config();
22117        runtime.child_roster = runtime
22118            .child_roster
22119            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
22120        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
22121        let pid_file = dir.join("grandchild.pid");
22122        let spec = ModuleSpec {
22123            script: None,
22124            module_id: module_id.to_string(),
22125            program: PathBuf::from("/bin/sh"),
22126            args: vec![
22127                "-c".into(),
22128                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
22129                "fixture".into(),
22130                pid_file.display().to_string(),
22131            ],
22132            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
22133                .into_iter()
22134                .map(|key| (key.to_string(), dir.display().to_string()))
22135                .collect(),
22136            reserved: false,
22137            reserved_prefixes: Vec::new(),
22138            protocol: ModuleProtocol::None,
22139            overlap: Default::default(),
22140            carry_containment: false,
22141        };
22142        let child =
22143            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
22144        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
22145        let grandchild: u32 = loop {
22146            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
22147                if let Ok(pid) = contents.trim().parse() {
22148                    break pid;
22149                }
22150            }
22151            assert!(
22152                tokio::time::Instant::now() < deadline,
22153                "grandchild pid was not recorded"
22154            );
22155            tokio::time::sleep(Duration::from_millis(10)).await;
22156        };
22157        assert!(
22158            running(grandchild),
22159            "grandchild must be alive before teardown"
22160        );
22161        if shutdown {
22162            let mut child = child;
22163            crate::child_roster::end_children_for_daemon_shutdown(
22164                &runtime.child_roster,
22165                false,
22166                std::future::pending(),
22167            )
22168            .await;
22169            child.wait().await.expect("reap shutdown straggler");
22170        } else {
22171            drain_child_to_state(
22172                module_id,
22173                ModuleProtocol::None,
22174                StopNotice::NotSent,
22175                &Registry::default(),
22176                None,
22177                &snapshot,
22178                &runtime.terminal_ring,
22179                &SpawnEventFeed::default(),
22180                child,
22181                Duration::from_millis(100),
22182                ModuleState::Stopped,
22183                Some(false),
22184                &FailureContext::operator(),
22185            )
22186            .await
22187            .expect("real supervisor teardown");
22188        }
22189        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
22190        while running(grandchild) && tokio::time::Instant::now() < deadline {
22191            tokio::time::sleep(Duration::from_millis(10)).await;
22192        }
22193        let survived = running(grandchild);
22194        // Kill a surviving grandchild so a failed test does not leave it behind.
22195        if survived {
22196            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
22197            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
22198            tokio::time::sleep(Duration::from_millis(100)).await;
22199        }
22200        if module.exists() {
22201            std::fs::remove_dir(&module).expect("remove empty module cgroup");
22202        }
22203        std::fs::remove_dir(root.join("subc-modules")).unwrap();
22204        std::fs::remove_dir(&root).unwrap();
22205        assert!(
22206            !survived,
22207            "grandchild {grandchild} outlived module teardown"
22208        );
22209        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
22210    }
22211}
22212
22213/// A slow live-children record write on the restart path must not stop the
22214/// daemon from answering the replacement module's authentication handshake.
22215#[cfg(all(test, unix))]
22216mod record_write_stall_tests {
22217    use super::*;
22218    use std::sync::atomic::{AtomicBool, Ordering};
22219    use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
22220
22221    /// `subc-client-rs` gives the whole handshake this long before the module
22222    /// exits with `Timeout { stage: ServerProof }`.
22223    const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
22224    /// How long the replacement's admission write is held: longer than the
22225    /// module's deadline, standing in for the delay a busy disk can add
22226    /// across the two syncs in that write.
22227    const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
22228
22229    /// A module that runs `/bin/sleep` directly, not through `/bin/sh -c`.
22230    /// On macOS every module is launched through the privacy trampoline, and
22231    /// a module that is not a `#!` script is admitted only if its running
22232    /// image is the configured program. With `/bin/sh -c "exec sleep 600"`
22233    /// the shell soon replaces itself with `/bin/sleep`, which does not match
22234    /// the configured `/bin/sh`. The image is read only after the spawn's
22235    /// live-children record write, and here that write lasted long enough for
22236    /// the shell to have exec'd every time, so each process was refused and
22237    /// killed right after spawn, and the supervisor crash-looped it with
22238    /// backoff until its restart budget ran out. The restart this test
22239    /// measures would then never produce a module at all.
22240    fn sleeping_spec() -> ModuleSpec {
22241        ModuleSpec {
22242            script: None,
22243            module_id: "record-write-stall".into(),
22244            program: "/bin/sleep".into(),
22245            args: vec!["600".into()],
22246            env: vec![],
22247            reserved: false,
22248            reserved_prefixes: vec![],
22249            protocol: ModuleProtocol::None,
22250            overlap: Default::default(),
22251            carry_containment: false,
22252        }
22253    }
22254
22255    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
22256    async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
22257        let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
22258        let supervisor =
22259            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
22260                .with_live_children_record(home.join("live-children.json"));
22261        let module = supervisor.spawn(sleeping_spec()).unwrap();
22262        // `reported_pid` names a process only once it is confirmed as the
22263        // module (on macOS, once the trampoline's image check has passed), so
22264        // a process the daemon is about to refuse and kill never counts as
22265        // the running module, here or in the final wait below.
22266        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
22267            loop {
22268                if let Some(pid) = lock_snapshot(&module.inner.snapshot)
22269                    .unwrap()
22270                    .reported_pid()
22271                {
22272                    return pid;
22273                }
22274                tokio::time::sleep(Duration::from_millis(10)).await;
22275            }
22276        })
22277        .await
22278        .expect("the first process was admitted as the module");
22279
22280        // The daemon's listener, served by this runtime's workers like the
22281        // supervisor's own tasks.
22282        let key = vec![0x42; 32];
22283        let daemon_id = [0x24; 16];
22284        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22285        let port = listener.local_addr().unwrap().port();
22286        let server = tokio::spawn(crate::server::serve_listener(
22287            listener,
22288            Arc::new(crate::router::Router::with_default_self_handler()),
22289            crate::server::ServerAuth::with_limits(
22290                key.clone(),
22291                daemon_id,
22292                "test",
22293                crate::server::DEFAULT_AUTH_DEADLINE,
22294                4,
22295            ),
22296        ));
22297
22298        // Hold the write that admits the replacement, once, and say when it
22299        // has started.
22300        let (held_tx, held_rx) = std::sync::mpsc::channel();
22301        let armed = AtomicBool::new(true);
22302        supervisor
22303            .child_roster
22304            .set_record_write_hook_for_test(move |live| {
22305                let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
22306                if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
22307                    let _ = held_tx.send(());
22308                    std::thread::sleep(HELD_RECORD_WRITE);
22309                }
22310            });
22311
22312        // The replacement module, as a separate process would be: its own
22313        // thread and runtime, so only the daemon's runtime can be stalled.
22314        let module_process = std::thread::spawn(move || {
22315            held_rx
22316                .recv_timeout(Duration::from_secs(30))
22317                .expect("the restart reached the replacement's record write");
22318            let conn = ConnectionInfo {
22319                schema: SCHEMA_VERSION,
22320                wire_version: None,
22321                endpoints: vec![Endpoint {
22322                    host: "127.0.0.1".to_owned(),
22323                    port,
22324                }],
22325                key,
22326                daemon_id,
22327                pid: std::process::id(),
22328                daemon_ver: "test".to_owned(),
22329            };
22330            tokio::runtime::Builder::new_current_thread()
22331                .enable_all()
22332                .build()
22333                .unwrap()
22334                .block_on(async {
22335                    let started = Instant::now();
22336                    let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
22337                        .await
22338                        .unwrap();
22339                    let result =
22340                        authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
22341                    (result, started.elapsed())
22342                })
22343        });
22344
22345        module.restart(Some(0)).await.unwrap();
22346        let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
22347            .await
22348            .unwrap();
22349        assert!(
22350            auth.is_ok(),
22351            "the daemon must answer a module's handshake while a restart waits on the \
22352             live-children record write; got {auth:?} after {elapsed:?}"
22353        );
22354
22355        tokio::time::timeout(Duration::from_secs(20), async {
22356            while lock_snapshot(&module.inner.snapshot)
22357                .unwrap()
22358                .reported_pid()
22359                .is_none_or(|pid| pid == old_pid)
22360            {
22361                tokio::time::sleep(Duration::from_millis(10)).await;
22362            }
22363        })
22364        .await
22365        .expect("the restart published its replacement");
22366        module.stop().await.unwrap();
22367        server.abort();
22368    }
22369}
22370
22371#[cfg(all(test, unix))]
22372mod retained_status_tests {
22373    use super::*;
22374
22375    #[tokio::test]
22376    async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
22377        use crate::live_children::{ContainerIdentity, RetainedReason};
22378        for (reason, carried, expected) in [
22379            (
22380                RetainedReason::ContainerSurvivors(vec![4242]),
22381                true,
22382                Some("waiting: container survivors (pids [4242])".to_owned()),
22383            ),
22384            (
22385                RetainedReason::ContainerUnverifiable("member inspection denied".into()),
22386                true,
22387                Some(
22388                    "waiting: container survivors could not be verified: member inspection denied"
22389                        .to_owned(),
22390                ),
22391            ),
22392            (RetainedReason::Survived, false, None),
22393        ] {
22394            let supervisor =
22395                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
22396            let spec = ModuleSpec {
22397                script: None,
22398                module_id: "retained-status-fixture".into(),
22399                program: PathBuf::from("/not-spawned"),
22400                args: Vec::new(),
22401                env: Vec::new(),
22402                reserved: false,
22403                reserved_prefixes: Vec::new(),
22404                protocol: ModuleProtocol::None,
22405                overlap: ModuleOverlap::Exclusive,
22406                carry_containment: carried,
22407            };
22408            let module = supervisor
22409                .supervise_configured(spec.clone(), false)
22410                .unwrap();
22411            // No process is launched. Invalid group zero is deliberate: if a
22412            // status read wrongly rechecks, verification changes the reason
22413            // and fails the assertions without signalling any real process.
22414            let guard = supervisor
22415                .child_roster
22416                .admit(
22417                    spec.module_id,
22418                    0,
22419                    ModuleProtocol::None,
22420                    None,
22421                    crate::child_roster::RecordedIdentity {
22422                        container: carried.then_some(ContainerIdentity {
22423                            pgid: Some(0),
22424                            created_start_time: Some(0),
22425                            cgroup_path: None,
22426                            ..ContainerIdentity::test_metadata()
22427                        }),
22428                        ..crate::child_roster::RecordedIdentity::default()
22429                    },
22430                )
22431                .unwrap();
22432            guard.retain(reason);
22433            let status = module.status().unwrap();
22434            assert_eq!(status.waiting_for_previous_detail, expected.clone());
22435            assert_eq!(status.health.detail, expected);
22436            assert!(!status.process_alive);
22437        }
22438    }
22439}
22440
22441fn registration_child_is_running(child: &mut SupervisedChild) -> bool {
22442    child.try_wait().is_ok_and(|status| status.is_none())
22443}